{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayes by backpropagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "notebook credit: [Nitarshan Rajkumar](https://www.nitarshan.com/bayes-by-backprop/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchvision import datasets, transforms\n",
    "from torchvision.utils import make_grid\n",
    "from tqdm import tqdm, trange\n",
    "\n",
    "from tensorboardX import SummaryWriter\n",
    "writer = SummaryWriter('/home/docker_user/runs/bnn')\n",
    "\n",
    "# sns.set()\n",
    "# sns.set_style(\"dark\")\n",
    "# sns.set_palette(\"muted\")\n",
    "# sns.set_color_codes(\"muted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "LOADER_KWARGS = {'num_workers': 1, 'pin_memory': True} if torch.cuda.is_available() else {}\n",
    "print(torch.cuda.is_available())\n",
    "print(torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to ./fmnist/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "26427392it [00:06, 4063166.39it/s]                              \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./fmnist/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "32768it [00:00, 199163.22it/s]           \n",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to ./fmnist/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n",
      "Extracting ./fmnist/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n",
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to ./fmnist/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4423680it [00:01, 2590511.84it/s]                              \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./fmnist/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8192it [00:00, 102299.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to ./fmnist/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n",
      "Extracting ./fmnist/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n",
      "Processing...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 100\n",
    "TEST_BATCH_SIZE = 5\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.FashionMNIST(\n",
    "        './fmnist', train=True, download=True,\n",
    "        transform=transforms.ToTensor()),\n",
    "    batch_size=BATCH_SIZE, shuffle=True, **LOADER_KWARGS)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.FashionMNIST(\n",
    "        './fmnist', train=False, download=True,\n",
    "        transform=transforms.ToTensor()),\n",
    "    batch_size=TEST_BATCH_SIZE, shuffle=False, **LOADER_KWARGS)\n",
    "\n",
    "TRAIN_SIZE = len(train_loader.dataset)\n",
    "TEST_SIZE = len(test_loader.dataset)\n",
    "NUM_BATCHES = len(train_loader)\n",
    "NUM_TEST_BATCHES = len(test_loader)\n",
    "\n",
    "CLASSES = 10\n",
    "TRAIN_EPOCHS = 5 #20\n",
    "SAMPLES = 2\n",
    "TEST_SAMPLES = 10\n",
    "\n",
    "assert (TRAIN_SIZE % BATCH_SIZE) == 0\n",
    "assert (TEST_SIZE % TEST_BATCH_SIZE) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modelling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Calculating the expectation of the likelihood over the variational posterior is computationally prohibitive, so we (once more) will rely on an approximate method to determining it. We approximate our cost function using sampled weights:</p>\n",
    "\n",
    "$\\mathcal{F}(\\mathcal{D}, \\theta) \\approx \\sum_{i=1}^n{\\log{q(\\mathbf{w}^{(i)}\\vert\\theta)} - \\log{P(\\mathbf{w}^{(i)})} - \\log{P(\\mathcal{D}\\vert\\mathbf{w}^{(i)})}}$\n",
    "\n",
    "<p>When using automatic differentiation as provided by frameworks such as PyTorch, we only need to worry about implementing this sampling, and setting up the cost function as above, and can leverage our usual backpropagation methods to train a model. The beauty of this algorithm is that it is a mathematically sound approximation for</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reparameterized Gaussian\n",
    "\n",
    "$$\\begin{aligned}\n",
    "\\theta &= (\\mu, \\rho)\\\\\n",
    "\\sigma &= \\log{(1+e^\\rho)}\\\\\n",
    "\\mathcal{N}(x\\vert \\mu, \\sigma) &= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n",
    "\\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n",
    "P(\\mathbf{w}) &= \\prod_j{\\mathcal{N}(\\mathbf{w}_j \\vert 0, \\sigma^2)}\\\\\n",
    "\\log{P(\\mathbf{w})} &= \\sum_j{\\log{\\mathcal{N}(\\mathbf{w}_j \\vert 0, \\sigma^2)}}\\\\\n",
    "\\end{aligned}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Gaussian(object):\n",
    "    def __init__(self, mu, rho):\n",
    "        super().__init__()\n",
    "        self.mu = mu\n",
    "        self.rho = rho\n",
    "        self.normal = torch.distributions.Normal(0,1)\n",
    "    \n",
    "    @property\n",
    "    def sigma(self):\n",
    "        return torch.log1p(torch.exp(self.rho))\n",
    "    \n",
    "    def sample(self):\n",
    "        epsilon = self.normal.sample(self.rho.size()).to(DEVICE)\n",
    "        return self.mu + self.sigma * epsilon\n",
    "    \n",
    "    def log_prob(self, input):\n",
    "        return (-math.log(math.sqrt(2 * math.pi))\n",
    "                - torch.log(self.sigma)\n",
    "                - ((input - self.mu) ** 2) / (2 * self.sigma ** 2)).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gaussian Scale Mixture Prior</h3>\n",
    "<p>The authors suggest the use of a scaled mixture of two gaussians for the prior distribution on weights. Both gaussians are zero mean, but with separate variances. When setting $\\sigma_1 \\gt \\sigma_2$ and $\\sigma_2 \\ll 1$, we obtain a spike-and-slab prior that combines a heavy tail ($\\sigma_1$) with a concentration around the 0 mean ($\\sigma_2$). Accordingly, this distribution’s probability function is as follows:</p>\n",
    "\n",
    "$$\n",
    "\\begin{aligned}\n",
    "P(\\mathbf{w}) &= \\prod_j{\\pi \\mathcal{N}(\\mathbf{w}_j \\vert 0, \\sigma_1^2) + (1-\\pi) \\mathcal{N}(\\mathbf{w}_j \\vert 0, \\sigma_2^2)}\\\\\n",
    "\\log{P(\\mathbf{w})} &= \\sum_j{\\log{(\\pi \\mathcal{N}(\\mathbf{w}_j \\vert 0, \\sigma_1^2) + (1-\\pi) \\mathcal{N}(\\mathbf{w}_j \\vert 0, \\sigma_2^2))}}\\\\\n",
    "\\end{aligned} $$\n",
    "\n",
    "<p>As the prior parameters are fixed and will not be modified in training, we don’t need to use a reparameterized Gaussian here:</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ScaleMixtureGaussian(object):\n",
    "    def __init__(self, pi, sigma1, sigma2):\n",
    "        super().__init__()\n",
    "        self.pi = pi\n",
    "        self.sigma1 = sigma1\n",
    "        self.sigma2 = sigma2\n",
    "        self.gaussian1 = torch.distributions.Normal(0,sigma1)\n",
    "        self.gaussian2 = torch.distributions.Normal(0,sigma2)\n",
    "    \n",
    "    def log_prob(self, input):\n",
    "        prob1 = torch.exp(self.gaussian1.log_prob(input))\n",
    "        prob2 = torch.exp(self.gaussian2.log_prob(input))\n",
    "        return (torch.log(self.pi * prob1 + (1-self.pi) * prob2)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAD8CAYAAACsP5F0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdeXxU5dn/8c8VVllEQEQBlaAgEJYAARcqIIg7ID7aqtiKiAqKS1ta0fq4Vmst1K3+EAQXrLvWShVREUXxEUlYBAEtKFRTkE0QF5Ys9++PexIDTJJJMjNnlu/79cpr5pxznzNXArlzrnNv5pxDREREREQkmWQEHYCIiIiIiEhVKZEREREREZGko0RGRERERESSjhIZERERERFJOkpkREREREQk6SiRERERERGRpBNRImNmp5nZZ2a2xswmhDk+0sw2m9nS0Nfo6IcqIsnIzB41s01m9sk++68O1SsrzOyeMvtvCNU1n5nZqfGPWESSkZn9OlSffGJmz5hZfTPLNLOPzGy1mT1nZnWDjlNEoqfSRMbMagEPAacDnYELzKxzmKLPOeeyQ1/TohyniCSvx4HTyu4ws5OAYUA351wWMDG0vzNwPpAVOuf/heogEZFymVlr4BogxznXBaiFr0v+DNzrnGsPbAMuDS5KEYm2SFpk+gBrnHNfOOf2AM/ib0BERCrlnHsP+Gaf3WOBu51zu0NlNoX2DwOedc7tds6tBdbg6yARkcrUBg4ws9pAA2ADMBB4MXT8CeDsgGITkRioHUGZ1sBXZbbzgWPDlPsfM+sH/Bv4tXPuq30LmNnlwOUADRs27NWxY8eqRyyJZdEi/9qrV7BxSFQsWrRoi3OuRRw+qgNwopndCewCxjvncvH1zYIy5fJD+/aj+kQkscWxPsE5918zmwh8CewE3gQWAdudc4WhYqpP0sTqb1azY9cOjm5+NE3qNQk6HKmhiuqSSBIZC7PP7bP9L+AZ59xuMxuDf+oxcL+TnJsKTAXIyclxeXl5EXy8JDQL/ffQv2VKMLP/xOmjagNNgeOA3sDzZtaOyOobv1P1iUhCi2N9gpk1xbfoZgLbgRfwXeL3pfokDZz+1OnMXjObBy58gNPbh/tvIMmkorokkq5l+cDhZbbbAOvLFnDObS3pIgI8AujxvIhUJB/4h/MWAsXAwURQ30jqWrd9HQVFBUGHIcnpZGCtc26zc64A+AdwAnBQqKsZqD5JG603/MAbMyBj566gQ5EYiySRyQXah2b+qIsfPDezbAEzO6zM5lBgVfRCFJEU9E9CrbZm1gGoC2zB1y3nm1k9M8sE2gMLA4tS4mbrj1vJvD+Tq1+/OuhQJDl9CRxnZg3MzIBBwErgHeDcUJmLgVcCik/i6LKnP+WUL+DgDz8OOhSJsUq7ljnnCs1sHPAGfhaQR51zK8zsdiDPOTcTuMbMhgKF+EG9I2MYs4gkETN7BhgAHGxm+cAtwKPAo6EpmfcAFzvnHLDCzJ7H34AUAlc554qCiVziafuu7QC89cVbAUciycg595GZvQgsxtcdS/BdxV4DnjWzP4b2TQ8uShGJtkjGyOCcmwXM2mffzWXe3wDcEN3QRKqvoKCA/Px8du1Ss3I49evXp02bNtSpUyfmn+Wcu6CcQxeVU/5O4M7YRSRSNapPKhbP+qQizrlb8A9KyvoCzXwoCUT1SfmqU5dElMiIJJv8/HwaN25M27ZtMQs3fjx9OefYunUr+fn5ZGZmBh2OSMJTfVI+1SeSyFz4uR0CpfokvOrWJZGMkRFJOrt27aJ58+aqJMIwM5o3b66nQSIRUn1SPtUnIlWj+iS86tYlSmQkZamSKJ9+NiJVo9+Z8ulnI1I1+p0Jrzo/FyUyIiISuETsAiIiyclUnaQNJTIicTR69GhWrlwZ088YMWIExxxzDF26dGHUqFEUFGhdDkkeFnZNVAknHvXJ3/72N44++mjMjC1btsT0s0QkGMl8b6JERiSOpk2bRufOnWP6GSNGjODTTz9l+fLl7Ny5k2nTpsX080SiSS0zkYtHfdK3b1/mzJnDkUceGdPPEYkmp+chVZLM9yZKZERi5IcffuDMM8+ke/fudOnSheeee44BAwaQl5cHwPTp0+nQoQMDBgzgsssuY9y4cQCMHDmSsWPHctJJJ9GuXTvmzZvHqFGj6NSpEyNHjiy9/tixY8nJySErK4tbbvlpxtEzzjgDM8PM6NOnD/n5+XH9vkWqQy0xFQuqPunRowdt27aN57cqEjV+eTIpK9XuTTT9sqS862Zfx9Kvl0b1mtmHZnPfafdVWGb27Nm0atWK1157DYBvv/2WyZMnA7B+/XruuOMOFi9eTOPGjRk4cCDdu3cvPXfbtm3MnTuXmTNnMmTIED744AOmTZtG7969Wbp0KdnZ2dx55500a9aMoqIiBg0axLJly+jWrVvpNQoKCnjyySe5//77o/q9i6Sz666DpdGtTsjOhvsqrk4Cr09EkktyPBgJ4v4k6Lok2vcmapERiZGuXbsyZ84crr/+et5//32aNGlSemzhwoX079+fZs2aUadOHc4777y9zh0yZAhmRteuXWnZsiVdu3YlIyODrKws1q1bB8Dzzz9Pz5496dGjBytWrNivf+uVV15Jv379OPHEE2P+vYpIbAVdn4hIagi6Lon2vYlaZCTlVdZyEisdOnRg0aJFzJo1ixtuuIFTTjml9Fhlzd316tUDICMjo/R9yXZhYSFr165l4sSJ5Obm0rRpU0aOHLnX3Ou33XYbmzdvZsqUKVH+rkTSW2UtJ7ESZH0ikmwsSbqUBXF/kmr3JmqREYmR9evX06BBAy666CLGjx/P4sWLS4/16dOHefPmsW3bNgoLC3nppZeqdO0dO3bQsGFDmjRpwsaNG3n99ddLj02bNo033niDZ555howM/YpLctAg/4oFVZ+IJDWt17KfVLs3UYuMSIwsX76c3/3ud2RkZFCnTh0mT57M+PHjAWjdujU33ngjxx57LK1ataJz5857Ne9Wpnv37vTo0YOsrCzatWtH3759S4+NGTOGI488kuOPPx6Ac845h5tvvjm635xIjGjQf3hB1ScPPPAA99xzD19//TXdunXjjDPO0EyIkjySpGUmnlLt3sSCmtEhJyfHlcyQIEms5GlHglUWq1atolOnTkGHUaHvv/+eRo0aUVhYyPDhwxk1ahTDhw+P2+eH+xmZ2SLnXE7cgogS1SfJb803a2j/YHvaNW3H59d8HnQ4e1F9UjnVJ5JIPup5CMcu2Uzu5P+l95jbgw5nL4lenyRbXaJ+JyIBufXWW8nOzqZLly5kZmZy9tlnBx2SSGDUElMzqk9EJBqSrS5R1zKRgEycODHoEEQkRag+ESlLD0aqK9nqErXIiIiIiEjKSJZZy6TmlMiIiEjgNGuZiESdZi1LeUpkREQkYWisjIiIREqJjIjElJk9amabzOyTMMfGm5kzs4ND22ZmD5jZGjNbZmY94x+xiCQbMzvGzJaW+dphZteZWTMze8vMVodemwYdq4hEjxIZkQDNnDmTu+++O+gwYu1x4LR9d5rZ4cBg4Msyu08H2oe+LgcmxyE+kaQ0evRoVq5cGdPPGDFiBMcccwxdunRh1KhRFBQUxPTzqss595lzLts5lw30An4EXgYmAG8759oDb4e2JU0EtcRIKkiW+xMlMiIBGjp0KBMmpPbfVefce8A3YQ7dC/we9hocMQyY4bwFwEFmdlgcwpQEobEykZs2bRqdO3eO6WeMGDGCTz/9lOXLl7Nz585kWQxzEPC5c+4/+DrlidD+J4DEnktWokNjY2osWe5PlMiIxMi6devo2LEjo0ePpkuXLowYMYI5c+bQt29f2rdvz8KFC3n88ccZN24cAMOGDWPGjBkATJkyhREjRgQZfkyZ2VDgv865j/c51Br4qsx2fmifpDiNjanYDz/8wJlnnkn37t3p0qULzz33HAMGDKBk4cbp06fToUMHBgwYwGWXXVZar4wcOZKxY8dy0kkn0a5dO+bNm8eoUaPo1KkTI0eOLL3+2LFjycnJISsri1tuuaV0/xlnnIGZYWb06dOH/Pz8uH7f1XQ+8EzofUvn3AaA0OshgUUl8aPnIRVKpfsTrSMjqe+662Dp0uheMzsb7ruv0mJr1qzhhRdeYOrUqfTu3Zunn36a+fPnM3PmTO666669FpqaOnUqffv2JTMzk0mTJrFgwYLoxpwgzKwB8AfglHCHw+wL+yfJzC7Hdz/jiCOOiFp8EoxkaYm5bvZ1LP06uvVJ9qHZ3HdaxfXJ7NmzadWqFa+99hoA3377LZMn+56X69ev54477mDx4sU0btyYgQMH0r1799Jzt23bxty5c5k5cyZDhgzhgw8+YNq0afTu3ZulS5eSnZ3NnXfeSbNmzSgqKmLQoEEsW7aMbt26lV6joKCAJ598kvvvvz+q33u0mVldYChwQxXPU32SihL9+YjuT2pMLTIiMZSZmUnXrl3JyMggKyuLQYMGYWZ07dqVdevW7VW2ZcuW3H777Zx00klMmjSJZs2aBRN07B0FZAIfm9k6oA2w2MwOxbfAHF6mbBtgfbiLOOemOudynHM5LVq0iHHIEi9qmQmva9euzJkzh+uvv57333+fJk2alB5buHAh/fv3p1mzZtSpU4fzzjtvr3OHDBlSWu+0bNlyrzqppB56/vnn6dmzJz169GDFihX7jb258sor6devHyeeeGLMv9caOh1Y7JzbGNreWNI9NfS6KdxJqk8k3aTK/YlaZCT1RfBkIlbq1atX+j4jI6N0OyMjg8LCwv3KL1++nObNm7N+fdh795TgnFtOme4doWQmxzm3xcxmAuPM7FngWODbkm4hIomgspaTWOnQoQOLFi1i1qxZ3HDDDZxyyk8NmpUNaC5b7+xbJxUWFrJ27VomTpxIbm4uTZs2ZeTIkezatau03G233cbmzZuZMmVKlL+rmLiAn7qVAcwELgbuDr2+EkRQImHp/qTG1CIjkiAWLlzI66+/zpIlS5g4cSJr164NOqSoMLNngA+BY8ws38wuraD4LOALYA3wCHBlHEIUSXjr16+nQYMGXHTRRYwfP57FixeXHuvTpw/z5s1j27ZtFBYW8tJLL1Xp2jt27KBhw4Y0adKEjRs38vrrr5cemzZtGm+88QbPPPMMGRmJfcsQ6rY6GPhHmd13A4PNbHXoWOJPwyRRo0nLoiOR70/UIiOSAHbv3s1ll13GY489RqtWrZg0aRKjRo1i7ty5WJLPvuKcu6CS423LvHfAVbGOSSTZLF++nN/97ndkZGRQp04dJk+ezPjx4wFo3bo1N954I8ceeyytWrWic+fOe3U9q0z37t3p0aMHWVlZtGvXjr59+5YeGzNmDEceeSTHH388AOeccw4333xzdL+5KHHO/Qg032ffVvwsZpJGkvzPZkJJ9PsTC2qO7ZycHFcy24oksZL/xAn22GPVqlV06tQp6DASWrifkZktcs7lBBRStak+SX5rvllD+wfb065pOz6/5vOgw9lLMtQn33//PY0aNaKwsJDhw4czatQohg8fHrfPV30iiSS3Z0t6L9nER5P/l2PH3B50OHtJhvokSFWtSxK7nVhERNKCBvnXzK233kp2djZdunQhMzNzrxmHRERSlbqWiYhI4JJl+uVENXHixKBDEBGJO7XIiIhIwlDLjIiIREqJjIiIiIikjAQbtisxpERGRERERESSjhIZEREREUkZ6qCaPpTIiMTR6NGjWblyZUw/429/+xtHH300ZsaWLVti+lkikjhmzpzJ3XdrvUcRqZpkvjeJKJExs9PM7DMzW2NmEyood66ZOTNLunnjReJh2rRpdO7cOaaf0bdvX+bMmcORRx4Z088RkcQydOhQJkwo90+0SPpQk0yVJPO9SaWJjJnVAh4CTgc6AxeY2X7frZk1Bq4BPopqhCJJ6ocffuDMM8+ke/fudOnSheeee44BAwZQstDa9OnT6dChAwMGDOCyyy5j3LhxAIwcOZKxY8dy0kkn0a5dO+bNm8eoUaPo1KkTI0eOLL3+2LFjycnJISsri1tuuaV0f48ePWjbtm08v1URibF169bRsWNHRo8eTZcuXRgxYgRz5syhb9++tG/fnoULF/L444+X1iPDhg1jxowZAEyZMoURI0YEGb6IJIhUuzeJZB2ZPsAa59wXAGb2LDAM2LcN6g7gHmB8VCOUhPXf/8IbXEJtCjltExxySNARhXfddbB0aXSvmZ0N991XcZnZs2fTqlUrXnvtNQC+/fZbJk+eDMD69eu54447WLx4MY0bN2bgwIF079699Nxt27Yxd+5cZs6cyZAhQ/jggw+YNm0avXv3ZunSpWRnZ3PnnXfSrFkzioqKGDRoEMuWLaNbt27R/UZFZG9BVSjAmjVreOGFF5g6dSq9e/fm6aefZv78+cycOZO77rprr0Uwp06dSt++fcnMzGTSpEksWLAgujGLSI0FUZ2k2r1JJF3LWgNfldnOD+0rZWY9gMOdc69WdCEzu9zM8swsb/PmzVUOVhKDc/CXv0DbtnApj3IxM8jMhGnTgo4ssXTt2pU5c+Zw/fXX8/7779OkSZPSYwsXLqR///40a9aMOnXqcN555+117pAhQzAzunbtSsuWLenatSsZGRlkZWWxbt06AJ5//nl69uxJjx49WLFiRcz7t4pIsDIzM/eqCwYNGlRaT5TUCyVatmzJ7bffzkknncSkSZNo1qxZMEGLBErzMO8r1e5NImmRCdfTsPR/hpllAPcCIyu7kHNuKjAVICcnR/+7ktQdd8Att8D//A/c/lJn9lCX3/ddymWXQUEBjB0bdIR7i+BBZ0x06NCBRYsWMWvWLG644QZOOeWU0mOukknu69WrB0BGRkbp+5LtwsJC1q5dy8SJE8nNzaVp06aMHDmSXbt2xeYbEZGfBFWhwH51Qdl6orCwcL/yy5cvp3nz5qxfvz5uMYokhCS5wwyiOkm1e5NIWmTygcPLbLcBytaKjYEuwLtmtg44DpipAf+p6c03fRLzq1/B889DZ1aRzcfMmgVnnQVXXw0LFwYdZWJYv349DRo04KKLLmL8+PEsXry49FifPn2YN28e27Zto7CwkJdeeqlK196xYwcNGzakSZMmbNy4kddffz3a4YtIElu4cCGvv/46S5YsYeLEiaxduzbokETizplG/e8r1e5NIklkcoH2ZpZpZnWB84GZJQedc9865w52zrV1zrUFFgBDnXN5MYlYAvP99zBqFHTqBFOmQEaZ/z21a8Pf/w6HHQaXXgp79gQXZ6JYvnw5ffr0Ke0zetNNN5Uea926NTfeeCPHHnssJ598Mp07d96rebcy3bt3p0ePHmRlZTFq1Cj69u1beuyBBx6gTZs25Ofn061bN0aPHh3V70tEEtvu3bu57LLLePTRR2nVqhWTJk1i1KhRlT5tFZHUl3L3Js65Sr+AM4B/A58Dfwjtux2fsOxb9l0gp7Jr9urVy0lyueMO58C5Dz4os9MPmSnd/Oc//ebkyfGPr6yVK1cGG0AEvvvuO+eccwUFBe6ss85y//jHP+L6+eF+RkCei6BOSLQv1SfJb/XW1Y5bcUfdf1TQoewnGeqToKk+kUSyMPsQ58AtmHxT0KHsJ9Hrk2S7N4loHRnn3CznXAfn3FHOuTtD+252zs0MU3aAU2tMytm61Q/wP/tsOOGE8ssNHeqP33EH7NwZv/iS0a233kp2djZdunQhMzNzrxmHREREpJrUo6zaku3eJJLB/iI8/DDs2OETlIqYwR//CAMHwpNPwuWXxye+ZDRx4sSgQxAREREplWz3JhG1yEh6KyiAyZNh8GDo0qXy8gMG+HnMH3zQ9zsLilN/8HLpZyNSNfqdKZ9+NpJwQv8lE/X/ZqLGFbTq/FyUyEil/vlPv/jlNddEVt7Mz172yScwb15sYytP/fr12bp1qyqLMJxzbN26lfr168fl88zsUTPbZGaflNn3FzP71MyWmdnLZnZQmWM3mNkaM/vMzE6NS5AiFVB9Ur541ycVMbODzOzFUN2yysyON7NmZvaWma0OvTYNOk6JvdKeZQk4a5nqk/CqW5eoa5lU6pFH/OKXp58e+TkXXAC//S1Mn+5baOKtZGYMLbwaXv369WnTpk28Pu5x4G/AjDL73gJucM4VmtmfgRuA682sM35mxCygFTDHzDo454riFazIvlSfVCzO9UlF7gdmO+fODc2y2gC4EXjbOXe3mU0AJgDXBxmkpDfVJ+WrTl2iREYq9PXX8PbbcOONUKtW5OcdcAD8/Od+SubJk6FRo9jFGE6dOnXIzMyM74dKWM6598ys7T773iyzuQA4N/R+GPCsc243sNbM1gB9gA/jEKoEKJGfTqo+SXxmdiDQj9Di3M65PcAeMxsGDAgVewI/s6oSGQmM6pPoUtcyqdCzz0JxMYwYUfVzL7oIfvwRXnkl+nFJShkFlKya1Rr4qsyx/NC+/ZjZ5WaWZ2Z5erKVOiwBu4JIUmgHbAYeM7MlZjbNzBoCLZ1zGwBCr4eEO1n1SYpK4AckEh1KZKRCTz0FPXtCx45VP7dvXzjySN8qIxKOmf0BKASeKtkVpljYv0TOuanOuRznXE6LFi1iFaLEWSK3zEhCqw30BCY753oAP+C7kUVE9UlqcXoekjaUyEi5Pv8c8vLgwgurd35GBvziFzBnDmzfHt3YJPmZ2cXAWcAI99Pdaz5weJlibYD18Y5N4k8tMVJD+UC+c+6j0PaL+MRmo5kdBhB63RRQfBJPeh6SNpTISLn+9S//Onx49a8xbBgUFsLrr1deVtKHmZ2G76c+1Dn3Y5lDM4HzzayemWUC7YGFQcQoIsnDOfc18JWZHRPaNQhYia9TLg7tuxhQZ+c0kMizlkl0abC/lGvmTMjKgnbtqn+NY4+Fli39OJkLLohebJI8zOwZ/GDbg80sH7gFP0tZPeCt0JP4Bc65Mc65FWb2PP4GpBC4SjOWiUiErgaeCs1Y9gVwCf6B7fNmdinwJXBegPGJSJQpkZGwtm2D996D3/++ZtepVQuGDIHnnoPdu6FevejEJ8nDORcuhZ1eQfk7gTtjF5EkIo2NkZpyzi0FcsIcGhTvWEQkPtS1TMKaPRuKinwSUlPDhsF338G779b8WiKS2jRWRkREIqVERsKaORMOOQT69Kn5tQYN8i0xb7xR82uJiIiIiIASGQmjuBjeegtOO61qi2CW54AD4MQT4c03Ky8rIulNXcxEJFqcKw46BIkxJTKyn48/hq1b4eSTo3fNU06BFStgvSbSFZEw1KVMRKJH9Um6UCIj+5kzx78OiuLwyMGD/etbb0XvmiIiIiL7C7Xs6gFJylMiI/t5+23o1AlatYreNbt1gxYtlMiIiIhIfJhaZlKeEhnZy+7dftrlaHYrA8jI8K0yc+b4MTgiImVpbIyIiFSVEhnZy4IFsHNndLuVlRg8GDZu9GNlRETC0VgZERGJlBIZ2cucOb71ZMCA6F+7f3//+v770b+2iIiIiKQXJTKyl3fegZwcaNIk+tdu2xbatPFd10RERERiyaEuq6lOiYyU2rULcnOhX7/YXN/MX/u990Dd4UUkHI2VEZGaUxfVdKFERkrl5cGePfCzn8XuM/r1gw0b4PPPY/cZIpJ8NDZGRESqSomMlJo/37+ecELsPqOktUfdy0SkLLXEiIhIVSmRkVLz50PHjn69l1jp2BEOPliJjIiEp5YZEak5PRhJF0pkBPBru3zwQWy7lcHe42REREREYkULYqY+JTICwKpVsH177BMZ8InM2rWQnx/7zxIREZH0pFnLUp8SGQF+Gh8Tj0SmZAzOggWx/ywRERFJL6XtMOqqmvKUyAjgE5lDD4V27WL/Wd27Q716SmREZH8a9C8iUaP6JOUpkRHAJzI/+1l8Hl7UrQu9esGHH8b+syR4ZvaomW0ys0/K7GtmZm+Z2erQa9PQfjOzB8xsjZktM7OewUUuIsnEzNaZ2XIzW2pmeaF9YesaSW1OY2PShhIZYcMGWLcO+vaN32cefzwsWuTXrZGU9zhw2j77JgBvO+faA2+HtgFOB9qHvi4HJscpRkkQmrVMaugk51y2cy4ntF1eXSMiKUCJjJCb61/79InfZx53HOzeDR9/HL/PlGA4594Dvtln9zDgidD7J4Czy+yf4bwFwEFmdlh8IhWRFFReXSMiKUCJjJCXBxkZkJ0dv8887jj/qu5laaulc24DQOj1kND+1sBXZcrlh/btx8wuN7M8M8vbvHlzTIMVkaTggDfNbJGZXR7aV15dsxfVJ6nFNFtZ2lAiI+TlQVYWNGgQv89s08Z/acC/7CNcv6Kwf5Gcc1OdcznOuZwWsVzFVUSSRV/nXE98F9WrzKxfpCeqPklNSmdSX0SJjJmdZmafhQbg7te/1MzGlBlgN9/MOkc/VIkF53wik5NTedloO+44JTJpbGNJl7HQ66bQ/nzg8DLl2gDr4xybiCQh59z60Osm4GWgD+XXNSKSAipNZMysFvAQ/glHZ+CCMInK0865rs65bOAe4K9Rj1Ri4ssvYfPmYBKZ44/3C2Nu3Bj/z5bAzQQuDr2/GHilzP5fhWYvOw74tqRbiIhIecysoZk1LnkPnAJ8Qvl1jaQBTR2S+iJpkekDrHHOfeGc2wM8ix88V8o5t6PMZkPUmpc08vL8a+/e8f/sknEyapVJbWb2DPAhcIyZ5ZvZpcDdwGAzWw0MDm0DzAK+ANYAjwBXBhCyBEjryEg1tQTmm9nHwELgNefcbMqvaySFafrl9FE7gjLhBt8eu28hM7sK+A1QFxgY7kKhwXeXAxxxxBFVjVViIC8P6tSBbt3i/9k9e/rPXrAAhg2rvLwkJ+fcBeUcGhSmrAOuim1Eksg0/bJUh3PuC6B7mP1bCVPXiEhqiKRFJqLBt865h5xzRwHXAzeFu5AG0yWevDzo2hXq1Yv/Z9ev7z+7pFVIREREpKY0a1n6iCSRqerg22fRPO1JIciB/iV69/YxqDeJiIC6lolI9DglNCkvkkQmF2hvZplmVhc4Hz94rpSZtS+zeSawOnohSqx8/jls3x5sIpOT42P4/PPgYhCR4KlLmYhEi1N9kjYqHSPjnCs0s3HAG0At4FHn3Aozux3Ic87NBMaZ2RkoRrMAACAASURBVMlAAbCNn2YIkQQW5ED/EiVJVF4eHH10cHGIiIiISHKJZLA/zrlZ+NmEyu67ucz7a6Mcl8RBXp4fG5OVFVwMWVl+rExuLpx/fnBxiEiw1KVMRKLNVK2kvIgWxJTUlJcH2dl+5rCg1KnjY9CAfxEREYkG5S/pQ4lMmiouhkWLgh0fU6J3bx9LUVHQkYhIUDRGRkSiRS0x6UOJTJr67DP4/vvESGRycuCHH3xMIiIiItHg9Hwk5SmRSVOJMNC/RNkB/yKS3jRdqojUWCiBUctM6lMik6by8qBBA+jYMehI4JhjoFEjP+BfRNKThV17WUREpHxKZNJUXh707Am1agUdiY+hZ0+1yIikM7XEiIhIVSmRSUOFhbBkSWKMjynRuzcsXQoFBUFHIiJBUsuMiESNpnVPeUpk0tDKlbBzZ2IlMjk5sGsXrFgRdCQiEiS1zIiISKSUyKShqA30j+KTjpKkSuNkRNKTWmJEJOo0rXvKUyKThvLy4MAD4eijg47kJ0cdBQcdpHEyIiIiIhIZJTJpKC8PevWCjAT61zfzrTJqkRFJb0592kUkWlSfpLwEupWVeNizBz7+OLHGx5TIyYFPPvFjZUQkvZi6gIhIlGghzPShRCbNLF/uk5lETWQKCnyMIpJe1BIjIiJVpUQmzURtoH8MlCRXGieTPszs12a2wsw+MbNnzKy+mWWa2UdmttrMnjOzukHHKfGjlhmpLjOrZWZLzOzV0LbqkjRlei6SNpTIpJm8PGjWDNq2DTqS/R1xBBx8sBKZdGFmrYFrgBznXBegFnA+8GfgXudce2AbcGlwUUq8qWVGauBaYFWZbdUlIilOiUyayc31LR+J+NCzZMC/Epm0Uhs4wMxqAw2ADcBA4MXQ8SeAswOKTeJILTFSE2bWBjgTmBbaNlSXpK2SMTJalyr1KZFJIzt3+sH0URsfE4Mnpzk5flHMH3+M+qUlwTjn/gtMBL7EJzDfAouA7c65wlCxfKB1uPPN7HIzyzOzvM2bN8cjZImxY7+C+nuKgw5DktN9wO+Bkv9AzYmwLgHVJ6lHD0bShRKZNPLxx1BUlJgD/Uvk5PgYP/446Egk1sysKTAMyARaAQ2B08MUDZsxO+emOudynHM5LVq0iF2gEhe1Nm1hwXSY+JRuIqVqzOwsYJNzblHZ3WGKlvv0TfVJatJYmdSnRCaNJPJA/xIa8J9WTgbWOuc2O+cKgH8AJwAHhbqaAbQB1gcVoMTR998DkJW/J+BAJAn1BYaa2TrgWXyXsvtQXZK2NP1y+lAik0by8qBlS2hdbuN68Fq1gkMPVSKTJr4EjjOzBqH+7IOAlcA7wLmhMhcDrwQUn4gkAefcDc65Ns65tvgJQ+Y650agukQk5SmRSSOJPNC/hAb8pw/n3Ef4gbiLgeX4+mgqcD3wGzNbg+/nPj2wICV+NFuZRJ/qkjRlqk/SRu3Ki0gq+P57WLUKzjsv6Egql5MDr73mY27UKOhoJJacc7cAt+yz+wugTwDhiEiSc869C7wbeq+6RCTFqUUmTSxZ4h94JvJA/xI5OT7WJUuCjkRE4iaRm4pFJKk41SdpQ4lMmijpqpUMiUyvXv5V3ctE0oi6gohIlCiNSR9KZNJEXh60aeMH0ie6Qw/1sSqRERERkWrTA5KUp0QmTZQM9I+qGFYQOTmwaFHl5UQkNWi6VBGJFqUv6UOJTBrYvh1Wr06ObmUlevWCzz6DHTuCjkRE4kEL14lItOi5SPpQIpMGFi/2r8mUyJTEWhK7iKQHtcyISE2VPhfRoP+Up0QmDSTTQP8SGvAvkp7UMiMiUaMxMilPiUwayM2FzExo3jzoSCLXogUceaQSGREREREJT4lMGsjLS67WmBI5OUpkRNKNupaJiEiklMikuC1bYN265E1kPv8ctm0LOhIRiTWnLiAiEm2qV1KeEpkUl4zjY0qUxKxpmEVERERkX0pkUlxurn8tGTwfVTF+0qEB/yIiIlJtmrUs5SmRSXF5eXDMMdCkSQwuXlwcg4v+pGlTOOooJTIiIiJSdU5LY6a8iBIZMzvNzD4zszVmNiHM8d+Y2UozW2Zmb5vZkdEPVaojNxd6947RxYuKYnThn2jAv0iaUF92EYmWUEuMpnNPfZUmMmZWC3gIOB3oDFxgZp33KbYEyHHOdQNeBO6JdqBSdf/9L2zYEMNEJsYtMuATmf/8BzZvjvlHiYiIiEgSiaRFpg+wxjn3hXNuD/AsMKxsAefcO865H0ObC4A20Q1TqqNkfExMBvp/+incccdP23/+s884okwD/kXSg6ZdFpFoUUNM+ogkkWkNfFVmOz+0rzyXAq+HO2Bml5tZnpnlbdYj9pjLzYVatSA7O4oX3bYNfvUr6NwZ7inT8DZhArRrB+PGwXffRe3jevb0r+peJpLi1LVMRESqKJJEJtxzsrB/cczsIiAH+Eu44865qc65HOdcTosWLSKPUqolNxe6dIEGDaJ0wc8+gx494Jln4Pe/hw0bsFvBbgW+/BLGjoX/9/+gb1/46qtKLhaZAw/0kxUokRERERGRsiJJZPKBw8tstwHW71vIzE4G/gAMdc7tjk54Ul3O+Zv/qI2PWbUK+vWDnTth/ny4+25o2fKn44cfDn/7G7z5pu9iduKJfpBOFGjAf+oys4PM7EUz+9TMVpnZ8WbWzMzeMrPVodemQccpcaBpUqUGzKy+mS00s4/NbIWZ3Rban2lmH4Xqk+fMrG7QsYpI9ESSyOQC7UOVQV3gfGBm2QJm1gOYgk9iNkU/TKmqzz/3vcCikshs2QJnneVvNN57D449tvyyJ58Mc+fC1q1w2mlR6WbWq9dPExdIyrkfmO2c6wh0B1YBE4C3nXPtgbdD25LqSrqWqYeZVM9uYKBzrjuQDZxmZscBfwbuDdUn2/Dd30UkRVSayDjnCoFxwBv4m4znnXMrzOx2MxsaKvYXoBHwgpktNbOZ5VxO4qRkoH+NE5miIvj5z30m8corvp9XZXr1gpdfhpUrYcyYGvd914D/1GRmBwL9gOkAzrk9zrnt+MlEnggVewI4O5gIJQga9C/V4bzvQ5t1Ql8OGIifTRVUn6SN0mpEY+9SXkTryDjnZjnnOjjnjnLO3Rnad7Nzbmbo/cnOuZbOuezQ19CKryixlpsL9ev7MTI1ct998M47fuxLRS0x+zr5ZLjtNnj6aXj88RqF0KOHbwxS97KU0w7YDDxmZkvMbJqZNQRaOuc2AIReDwl3siYPSU1a90Gqy8xqmdlSYBPwFvA5sD30QBYqmKxI9Ulq0QOR9BFRIiPJJzfXz1ZWp04NLrJyJfzhDzBsGFxySdXPv/FG6N8ffv3rGvULa9QIOnVSi0wKqg30BCY753oAP1CFbmSaPCS1OD05lRpyzhU557LxY3n7AJ3CFSvnXNUnKUQPRNKHEpkUVFgIixfXsFuZc34WskaNYMqU6g3EzciARx6BXbvg2mtrEMxPA/51r5NS8oF859xHoe0X8YnNRjM7DCD0qnF3aURPUqWmQl1U3wWOAw4ys9qhQ2EnK5IUpklEUp4SmRS0ahX8+GMNE5kXXvAD+++8c+/ZyaqqfXu4+WZ/vTlzqn2ZnBz4+mtYrz9BKcM59zXwlZmVDLwaBKzETyZycWjfxcArAYQnIknEzFqY2UGh9wcAJ+PH9b4DnBsqpvokTZQ+89TTz5SnRCYF1Xig/48/wu9+B927w+jRNQ/ot7+Ftm39NYuLq3WJkgH/GieTcq4GnjKzZfiZhu4C7gYGm9lqYHBoW0SkIocB74TqklzgLefcq8D1wG/MbA3QnNDkIpLi1BCTNmpXXkSSTW4uNG4MHTpU8wIPPugXuJwxA2rVqnlA9erBXXfBhRfC3/8Ov/pVlS/RvbsPJS/PD9mR1OCcW4pfRHdfg+IdiwTLad5lqQHn3DKgR5j9X+DHy4hIClKLTApasAD69PFDVKrsu+/gnnvg9NP9QP1o+cUvfLPKTTfB7qqvl9qgAWRl/dTaJCIiIlIhdS1LeUpkUsz338OyZXD88dW8wIMPwjffwK23RjMsn1XdeSd89RU88UTl5cPo3RsWLqx27zQRSWi64RARkapRIpNi8vL8jX61Eplvv4WJE+Gss3yTTrQNHuyv+6c/QUFBlU8/4QTYtg3+/e/ohyYiIiIpRrOWpTwlMinmww/9a1XWriz10EM+U4h2a0wJM9+1bN06v1BmFZUkZ//3f9ENS0QSgW44RESkapTIpJgPP/SD/Js3r+KJu3fDAw/AqadCr14RnVKtBezOOsuv1HnXXVBUVKVTjzkGmjb9KVkTkdShwf4iEnUaI5PylMikEOf8TX61upU99RRs3Ajjx0c9rr2YwY03+v5hr75apVMzMuC445TIiKQircQtItHiQi28qlZSnxKZFPL557BlSzUSGefgr3+Fbt1gUBxmvR0+HI48Eu69t8qnnnACrFgB27fHIC4RERERSRpKZFLIggX+tcqJzBtv+Oxg/Pj4DIyrXRuuvhrmzYMlS6p0asn39tFHMYhLRALjNERGRKJM1UrqUyKTQj780C+EmZVVxRMnToRWrfxaL/EyejQ0alTlVpmS9XHUvUwkxagvu4iIVJESmRTy4Yf+Rr9WrSqc9Mkn8PbbcM01ULduzGLbT5MmMGoUPPssbNgQ8WmNG0PXrpq5TERERCTdKZFJET/84BfCPO64Kp44ZQrUqweXXhqTuCp0zTVQWAiTJ1fptOOP913LtDCmSArReg8iIlJFSmRSRG6un824SuNjfvgBZsyAc8+Fgw+OWWzlOuooOOMMmDatSgtknnAC7NgBK1fGMDYRERFJbuqymvKUyKSI+fP9A80qJTLPPuszgjFjYhZXpcaO9V3LXnkl4lO0MKaIiIiUSw28aUOJTIp47z3o0gWaNavCSVOm+JkB+vaNWVyVOu00PxVzFbqXHXUUtGihREYklZQssGt6gioiNaR1qdKHEpkUUFjob+r79avCSYsW+f5oY8YE2ze9Vi244gqYOxc++yyiU8zgZz/zyZuIpAiNkRERkSpSIpMClizxw12qlMhMmQIHHAAXXRSzuCI2ahTUqQMPPxzxKf37w9q18NVXMYxLREREko7WpUofSmRSQEnLxIknRnjCjh3w9NNwwQVw0EExiytiLVvCOefA44/Dzp0RndK/v3+dNy92YYmIiEgyU0aT6pTIpID33oP27eGwwyI84amnfBNOkIP89zV2LGzfDs89F1Hxrl19DqZERkRERMLTYJlUp0QmyRUXw/vvV6FbmXN+YH3PnpCTE9PYqqRfP+jcOeJB/7Vq+RYoJTLJz8xqmdkSM3s1tJ1pZh+Z2Woze87M4rhSqwTF6YZDasDMDjezd8xslZmtMLNrQ/ubmdlbofrkLTNrGnSsEkeaPCTlKZFJcitWwLZtVUhkFiyA5cv9APtEGlxr5luIFi6ExYsjOqV/f1i92s/eLEntWmBVme0/A/c659oD24AAVmuVuAvVRwlUK0lyKQR+65zrBBwHXGVmnYEJwNuh+uTt0LaIpAglMknu/ff9a8SJzMMPQ+PGfnxMovnlL/0EBFOmRFRc42SSn5m1Ac4EpoW2DRgIvBgq8gRwdjDRSRD0/FSqwzm3wTm3OPT+O/zDkdbAMHw9AqpP0k8iPbCVmFAik+TmzYM2bfxSLJX65ht4/nk/U1njxjGPrcoOOgjOP99PRPDdd5UWz87234YSmaR2H/B7oDi03RzY7pwrDG3n429G9mNml5tZnpnlbd68OfaRSlxo/QepKTNrC/QAPgJaOuc2gE92gEPKOUf1SSpS17KUp0QmiRUX++VXBg6M8KHDjBmwa5fvVpaoxoyB77/3ExJUonZtv56MEpnkZGZnAZucc4vK7g5TNOxfIufcVOdcjnMup0WLFjGJUeLH6YZDosDMGgEvAdc553ZEep7qk9Ti1BKTNpTIJLFly2DLFhg0KILCzvluZccdB927xzy2auvdG3r08LFGcGPTvz+sWgUbN8YhNom2vsBQM1sHPIvvUnYfcJCZ1Q6VaQOsDyY8iSslMlJDZlYHn8Q85Zz7R2j3RjM7LHT8MGBTUPGJSPQpkUlic+b414gSmXnz4LPPEmvK5XBKBv1//DF89FGlxQcO9K9z58Y4Lok659wNzrk2zrm2wPnAXOfcCOAd4NxQsYuBVwIKUUSSRGh83XRglXPur2UOzcTXI6D6RCTlKJFJYnPmQKdO0DrsCIJ9TJnix6D8/OdR+/yYTZd6wQV+8MvDD1datGdPaNYM3nwzNqFIIK4HfmNma/BjZqYHHI/EkTqESDX1BX4JDDSzpaGvM4C7gcFmthoYHNoWkRRRu/Iikoh27/Yzll0aycS0mzbBSy/BlVf6WcESXePGfkKCxx6Dv/7VZyrlqFXLt0i9+abvmaJuscnJOfcu8G7o/RdAnyDjkQCoa5nUgHNuPuXnwZH0WxCRJKQWmSS1YAH8+COcfHIEhR99FAoKEr9bWVlXXOEnJpgxo9Kip5wC69fDypVxiEtEYkILYoqISFUpkUlSc+b41oiStVTKVVzsu5UNGAAdO8YjtOjo3t1PTBDBoP/Bg/3rW2/FIS4RiSlNvywiIpGKKJExs9PM7DMzW2Nm+62Ka2b9zGyxmRWa2bnhriHR9dZb0KcPNGlSScE334R165KrNabEmDF+goL33quw2JFHwjHHaJyMSFJT1zIREamiShMZM6sFPAScDnQGLjCzzvsU+xIYCTwd7QBlf1u2wMKFP7VEVOjhh6FFCxg+POZxRd3Pf+4nKIhg0P8pp8C77/qxQyIiIiJ6QJL6ImmR6QOscc594Zzbg1/vYVjZAs65dc65Zfy0OrfE0OzZ/nfzrLMqKZifD//6l58RoG7duMQWVQccACNH+okKNlU89f8pp8DOnfDBB/EJTUSiS/cbIhI1mvgnbUSSyLQGviqznR/aV2VmdrmZ5ZlZ3ubNm6tzCQFeew1atoRevSopOG2avzu4/PK4xBUTV1zhJyp47LEKiw0Y4HO1WbPiE5aIRJcpkxERkSqKJJEJl9dW6y+Oc26qcy7HOZfTokWL6lwi7RUW+haZM86AjIr+9QoK4JFH4LTTIDMzbvFFXceOPkuZMsVPXFCORo3gpJN8A5SIJC89SBURkUhFksjkA4eX2W4DrI9NOFKZ//s/2L4dzjyzkoKvvurnJE7GQf77uuIKWLu20mnJhg6Ff/8bPv00TnGJiIhI4lEDb9qIJJHJBdqbWaaZ1QXOB2bGNiwpz6uvQp06EQz0nzwZ2rTxTTfJbvhwP2FBJYP+hwzxrzP1v1Mk+ahrmYhEm1bJTnmVJjLOuUJgHPAGsAp43jm3wsxuN7OhAGbW28zygfOAKWa2IpZBp7PXXoN+/eDAAysotHKlb70YMwZq145bbDFTrx6MGuX7jf33v+UWO/xw6NlTiYxIMtKCmCISbRp7l/oiWkfGOTfLOdfBOXeUc+7O0L6bnXMzQ+9znXNtnHMNnXPNnXNZsQw6XX36qc9Rhg6tpOADD/ib/2Qe5L+vyy+HoiKYPr3CYkOH+u53mktCJDlpQUwRqSkXaohxSmRSXkSJjCSGl17yr+ecU0Ghb76BGTPgoot8d6xU0a4dnHqq7162Z0+5xYYO9T1UXnstjrGJSM3phkNEokQdytKHEpkk8uKLcMIJfuhLuR55xC+ocu21cYsrbq67DjZsgGefLbdIdjYccYT/WYmIiIhI6lIikyTWrIGlS+HccysoVFgIDz3k5yHu2jVuscXNqadCVhZMmlTu01sz+PnP4c03feOUiCQX9WkXkZpyapNJG0pkkkRJt7L/+Z8KCr38Mnz1VWq2xoDPUn7zG1i2DN5+u9xi55/vl9F5+eU4xiYiNaMERkREqkiJTJJ44QXo08d3mwrLOd9S0a4dnHVWXGOLqxEjoGVL/72Wo2dPOProCnugiYiIiEiSUyKTBD79FBYtgl/8ooJC774LH30E48dDrVrxCi3+6tWDceNg9mxYEX6WbzPfKjN3LmzcGOf4REREJDGooTflKZFJAjNm+NzkwgsrKPSnP/mWiksuiVtcgRk7Fg44oMJWmfPPh+Ji35IlIklAXctERKSKlMgkuKIiePJJP8790EPLKbRokV8A89e/hvr14xpfIJo3h0sv9T+YdevCFsnKgm7dfBIoicnMDjezd8xslZmtMLNrQ/ubmdlbZrY69No06Fgl9rQgptSEmT1qZpvM7JMy+1SXpDuN+U95SmQS3DvvQH4+XHxxBYX+9Cdo0sS3VMRRoAtNXX89ZGT4770cl14Kubnw8cdxjEuqohD4rXOuE3AccJWZdQYmAG8759oDb4e2JU3ovkOq6XHgtH32qS5Jd3o+kvKUyCS4J57wOcrQoeUUWLUK/vEPuOoqOPDAuMYWqDZtfKby2GPw5Zdhi4wYAXXrwvTpcY5NIuKc2+CcWxx6/x2wCmgNDAOeCBV7Ajg7mAglrkIPRnTfIdXhnHsP2HfSfdUl6UpPRNKGEpkEtnWrX9jxwgsr6DF2883QsKFfLDLdTAg9XLv77rCHmzeHc86Bv/8ddu2KY1xSZWbWFugBfAS0dM5tAJ/sAIcEF5mIJDHVJSIpTolMAnv0UX8DfuWV5RRYvNhnOr/5DbRoEdfYEsIRR/jJDaZP9/3vwhg9GrZt841WkpjMrBHwEnCdc25HFc673MzyzCxv8+bNsQtQ4srUJCMBUH2SqlShpDolMgmqqAgmT4Z+/aBLl3IK3XQTNG3qE5l0dcMNvkvKLbeEPXzSSXDUUfDgg3GOSyJiZnXwScxTzrmSdHOjmR0WOn4YsCncuc65qc65HOdcTot0TORTjCvWDYdEXUR1Cag+EUlWSmQS1OzZsHatH/oS1gcfwOuv++5VTZrENbaE0ratX1fmscdg+fL9DmdkwLXXwoIF8OGH8Q9PymdmBkwHVjnn/lrm0EygZHqLi4FX4h2biKQE1SUiKU6JTIK691447DAYPjzMweJi+O1vfYFx4+IeW8K56SafzP3+92EPX3KJP3zvvXGOSyrTF/glMNDMloa+zgDuBgab2WpgcGhb0oTG6Ep1mNkzwIfAMWaWb2aXorpEJOXVDjoA2d/ChfD22/CXv0CdOmEK/P3v8NFHfkqzBg3iHl/CadYM/vAH+N3vYM4cOPnkvQ43agRXXAETJ/plZ9q2DSRK2Ydzbj7l37cOimcsEjzTgphSA865C8o5pLokjWnMXepTi0wC+tOf/NCXK64Ic/C77/waKsceCxddFPfYEta4cT5Due462LNnv8NXX+27md1zT/xDE5HKaUFMEYkWp6bdtKFEJsGsWAH//Cdccw00bhymwJ13wtdfw/33+ztz8erXhwce8D/Av/51v8Mly85Mmwb/+U8A8YmIiIhIVOlOOMHcdJNPYK6+OszBpUt9/6hLLvEtMrK3IUP8oKLbb/czJezjD38AM/jjHwOITUQioq4gIlJTJfWIWnpTnxKZBDJ/vm+NmTDBL+a4l4ICGDXKrxczaVIg8SWF+++HWrX8dG/79Lk//HDfXe+xx2D16oDiE5HwNEZGRESqSIlMgnDOT7rVqpUf5rGfiRNhyRJ46CE/gEbCO/xwuOsuPzX1I4/sd/jGG+GAA9J76R0RERGRVKBEJkE8+aRf5+T228NMRJab6xd8PO88OOecQOJLKlddBYMHw69/Df/+916HDj0Ubr4ZXn0VZs0KKD4R2U9JFxCN0RWRmirtomqqUVKdEpkEsGWLbyE4/ng//GUv334Lv/iFXzNmypRA4ks6GRnw+ON+AoARI2D37r0OX3stHHOMf925M5gQRWQf6lkmIiJVpEQmAYwf7/OVqVP3mYjMORg9Gr78Ep55Rl3KqqJVKz9FWV6enzmhTP/7unV9D701a/wEACIiIpI69FwkfSiRCdjzz/t1La+/Hrp02efgbbfBiy/6hWVOOCGQ+JLa8OFwww1+rMzDD+91aNAg3wPtvvvg3XeDCU9ERESiz0pSGXUtS3lKZAK0di1cdpmfSfmWW/Y5+PTTPpG55BLfZCPVc8cdcMYZfmGe2bP3OvTnP8PRR8MvfwmbNgUUn4gAYJq1TESiTYlMylMiE5Dvv/9p3P4zz0CdOmUO/utfMHIk9O/vWxL0i1h9tWr5pLBrV/8Dnz+/9FDDhvDcc7B1q59HoaAgwDhF0pzWexCRqFF1kjaUyASgqAguuACWLYNnn4XMzDIHZ8+Gc8+F7Gx45RU/oENqpkkT/3M94gg480w/PVxIjx5+KM1778HYsVrKQiRoWhBTRGpOFUm6UCITZ0VFvrfYq6/Cgw/C6aeXOfj3v8PQoZCVBW+84W/AE1hSPUE95BB46y3/OmgQvPZa6aELL4T//V+YPt3PHqdkRiQAoV88JTIiUmOhesSpR0vKUyITRwUFcPHFfs2YO+6AK68MHSgu9gvI/PKX8LOfwdy5STFDmUu2O/7DD4cPPoBOnWDYMHjggdKbp9tu89Mx33effy0qCjhWkXQT+l0s1l8lEamxJLs/kWqrHXQA6WLLFt9jbN48PwnZhAmhAxs3wq9+BW++6ROZadOSpjtZUrXIlDjkEHjnHbjoIp+xfPABTJ2KNWnCvff6ITV//St89ZVPOBs1CjpgkTRRVAzo9kNEokeTiKQ+PfuKg3fegV69YMEC33tswgT808cZM/wg9Pfe84vIPPFE0iQxkIQtMiUOPBD++U+4+24/vXXnzvDyy5jBpElw//1+eFLPnn4ZGhGJPeeKgw5BRFJMUj5wlSpRIhNDmzb5AeQDB0K9evD++36hed57D/r18/3Mjj7a3y1fdlnSzU6W1BVERoZfvGfBAjj4YD+j2amnQl4e11zjk8+dO+H44+G3v/ULlopI7JQkMi65qkERSURJfHsiVaNEJga+/tqvC3PUUX4txuuug6W5BfTOfxkGDPDTKq9e7buRzZ/vB/cnoeJUeILau7dPJCdNgkWL/PaZMKHWLgAACpZJREFUZ9L/x9f5eEkxI0fCvfdC+/a+S+C2bUEHLJKaXLGvT5THiEjNub1eJHUpkYmSnTt9d6QLL/Sz/N5+OwweVMyKR/6Pe911NOjQxj/1X7PG911auxYuvdS3DCSppO1atq86dfx0ZV984Uf9L1oEZ5xBs95H8UiT8eQ+spSePRw33uj/bUeO9EOaCguDDlwkdbhiP8OGUyojIiIRiugu2sxOM7PPzGyNmU0Ic7yemT0XOv6RmbWNdqCJxDnfbWzuXPjjH/0Uygcf7Dj7bJj9rz2M7fEhn/5sNP+YexDHjOrrF7U84QQ/5/K6dX6V+QMOCPrbqLGk7loWzoEHws03w5df+gV+OnaEBx6g1+gezP6oKUt/No6fH/MxL79QwKmnQrNmjiFDfGPOO+/4hTUlOiqrcyT1lLTIqGuZRJvqk/RTXGTsoh57CmtRWKhlFVJZpbOWmVkt4CFgMJAP5JrZTOfcyjLFLgW2OeeONrPzgT8Dv4hFwBVxDoqLHK5476999+27XVAAP/7g2LkTfvwRftxRyM5v9/DDt4Vs2VjExo2wabOxcUst1m2oy783NGb7zvoAGMVk1V3DyMK5DOdF+n8/jzp5xdCli2+eOfNMP0imYcN4/zhiLiW6loVTty784hf+69tvYdYseOcdun/wDtNXPsRD1ON1TueNnWcx981BvPpq29JTDzvwe45s8SOHH7KHw1sXcehhGRzUvBYHHQQHNcugSbNa1G9Um3oNa1OvUR3qNahFvfpG3foZ1K1nWIZh5hvqzJJu2FRURFjnJI2yf0BL3rtit9frXvvcTwX33Vd6rZL3ZfaFvVa4a5Y5r7jIUVjopxsvLHAUFUNRIRQV+v3FRc7vLyimuLCYwj3FFBcUlW6XvJZ8lW4XFFFcVExxQTGuqNhPMV/myxUVY8VFe+2z9Z/QAzjs2yIeG7MAh/8lyKhlWK2MvV73fZ9Re+/jtWr/tK/keK3aVnrMMoxadcqULftaZ++ye70PHQv3ZRlp+MuaBFKhPin9HS/nd7683/dK65IK6pFw1ywq8stIFOxxFBSafy2AwiKjYHfxfl+Fe0Lv9zj27C5mz24o2F3Mnt2OPXtgz57Q627YU0Bon1FQAHsKbO+vQmNPQQZ7Co2CQmNPYUboqxZ7ijL4/+3db4wcdR3H8fdn/x5qW7X1b3tqQ4wBaRMNqX94AAKaqpVGowkYCJEHxEQSSCQG5InxiSQm6gNMlPgnREiIIkiVEIqBRxpNEUoTUttcCbFHsbVUWs/aXu/264OZu26ve3f7Z7Yzc/t5JZPZmZ29/XS799n57ezeTM9Wz06tGq04lPyD7konQAqqalGtBLVKMp+73Ky3aNZmGavP0qy3GKsn68YaLZqNuXkw1ohk3kzmzSaMjQWNhmjUg3pDNJqi0YBGU9SbSq5rJq/z9Ub6ej9Wod5MXvdrjQqVejWZapXz5+n+QKVydhrV/YNOuvnzy1uAiYh4GUDSw8B2oL0EtgPfSS8/AtwnSTHgZ48evPd3fP3uawk0P7WodFhuP7AkhvEp6zW8wbs4zAYmuYF9XKx9vOeivYyv2sXra09wYG2Fx94u7n1HhRfeW+VkYz+wH/Y8AHu6u49BHq5Bjo70e79nWmfmL6/63qq+778UNibT206+mUsOt/jQkSfZfOQJvvBGi9q/38mx45s4+L9N7D3xYQ6eGGf3gffxe8Y5xeBH3kRr/hlf6XC5823OXf/Eo3u48oufHDjLBdJN5/Tsl5+6mq/86VngbEO0n3yxvTXm1nda18vtKwvX9xp6RH3tp5/IO0JfWiRHlCKdt9ouz82HbdhvPL+yusllR08N+V4yNZQ++ehFf2b/qc3zH4Xsd975uqU+LLPyGqXONI1lpjpnaDDNmg7X1ZimWpmmpmkqOkO1Ns2bYprVM8k+ygw1ZqkyG+nUqjJLdX79DDVO05yfTjE2P59asHyasbblfD9Zk+wBt5bcFzi77eLX93tdFrfdevHj/GbixiW3W043A5n1wMG25UngY4ttExEzko4Da4Gj7RtJuhW4NV2ckrSvn9AZWMeCbMs5nk77gWcgebU4mU6HIXkJy1zPOXOyboqpUuRkwMdziuSJvvOcta+l084Ot+jLOTmDszsn/Z6n86ovdb3p+/u8iyx10zl99cktmcQ7T2l+T3HOrCyeMRbM85X9Y/n66V7eCl7RfTIkZXj+Q4Y5z6TTfwf5IZ13wVb0Yxkk+wQX8PzdmT+ejxwA6aZuNl20S7oZyHRqrIUV3c02RMT9wP1d3OdQSXouIi7PO8dynDNbzlka7pMhcM7slCEjlCfnkLlPhqAMOcuQEZxzUN182X8SGG9b3gAcWmwbSTVgDXAsi4BmNnK66Rwzs264T8xWsG4GMruAD0raKKkBXA/sWLDNDuDm9PKXgWcG/X6MmY2sbjrHzKwb7hOzFWzZj5al33m5DXgKqAK/iIiXJH0XeC4idgA/B34laYLkSMz1wwydgdwPH3fJObPlnCWwWOfkHGspZfn/cs7slCEjlCfn0LhPhqYMOcuQEZxzIPKBEzMzMzMzK5vynlbezMzMzMxGlgcyZmZmZmZWOiM/kJF0p6SQtC7vLJ1I+r6kv0vaI+kxSW/NO9McSVsl7ZM0IemuvPN0Imlc0rOS9kp6SdLteWdaiqSqpBck/SHvLNY790n/3CfZc5+UW5H7pMhdAu6TrBW5S0Z6ICNpHPg08I+8syzhaeCyiNhMcj7Ou3POAyRPauDHwGeBS4EbJF2ab6qOZoBvRsQlwMeBbxQ055zbgb15h7DeuU/65z4ZGvdJSZWgTwrZJeA+GZLCdslID2SAHwLfoijnYO4gInZGxEy6+BeSv4FfBFuAiYh4OSKmgYeB7TlnOk9EvBYRz6eX/0Pyi7g+31SdSdoAfB74Wd5ZrC/uk/65TzLmPim9QvdJgbsE3CeZKnqXjOxARtJ1wKsR8WLeWXpwC/Bk3iFS64GDbcuTFPAXsJ2kDwAfAf6ab5JF/YjkhauVdxDrjftkYO6T7LlPSqqEfVKkLgH3SdYK3SXLnkemzCT9EXh3h6vuAb4NfObCJupsqZwR8Xi6zT0khyEfupDZlqAO6wr5zhGApLcAvwXuiIgTeedZSNI24EhE/E3SVXnnsfO5T4bKfZIh90nxlaFPStol4D7JTBm6ZEUPZCLi2k7rJW0CNgIvSoLkkOjzkrZExD8vYERg8ZxzJN0MbAOuieKc+GcSGG9b3gAcyinLkiTVSUrioYh4NO88i7gCuE7S54AxYLWkByPixpxzWcp9MlTuk2y5TwquDH1S0i4B90mWCt8lPiEmIOkV4PKIOJp3loUkbQV+AFwZEf/KO88cSTWSL/hdA7wK7AK+WrQzJit5JXgAOBYRd+Sdpxvpux53RsS2vLNY79wnvXOfDI/7pNyK2idF7RJwnwxLUbtkZL8jUyL3AauApyXtlvSTvAMBpF/yuw14iuQLar8uWkmkrgBuAq5OH7/d6TsLZqPIfTIY94lZopBdAu6TUeMjMmZmZmZmVjo+ImNmZmZmZqXjgYyZmZmZmZWOBzJmZmZmZlY6HsiYmZmZmVnpeCBjZmZmZmal44GMmZmZmZmVjgcyZmZmZmZWOv8H93tfpocQb7MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "PI = 0.5\n",
    "SIGMA_1 = torch.cuda.FloatTensor([math.exp(-0)])\n",
    "SIGMA_2 = torch.cuda.FloatTensor([math.exp(-6)])\n",
    "\n",
    "def visualize_scale_mixture_components():\n",
    "    def show_lines():\n",
    "        pass\n",
    "    mix = ScaleMixtureGaussian(PI, SIGMA_1, SIGMA_2)\n",
    "    normal_1 = torch.distributions.Normal(0, SIGMA_1)\n",
    "    normal_2 = torch.distributions.Normal(0, SIGMA_2)\n",
    "    x_points = np.linspace(-5,5,10000)\n",
    "    d1 = np.array([torch.exp(normal_1.log_prob(float(c))) for c in x_points])\n",
    "    d2 = np.array([torch.exp(normal_2.log_prob(float(c))) for c in x_points])\n",
    "    d3 = np.array([torch.exp(mix.log_prob(float(c))) for c in x_points])\n",
    "    plt.subplots(1,3,figsize=(14,4))\n",
    "    plt.subplot(1,3,1)\n",
    "    plt.plot(x_points,d2,color=\"g\")\n",
    "    plt.plot(x_points,d3,color=\"r\")\n",
    "    plt.plot(x_points,d1,color=\"b\")\n",
    "    plt.legend([\"sigma2\", \"mix\", \"sigma1\"])\n",
    "    plt.ylim(0,0.5)\n",
    "    plt.subplot(1,3,2)\n",
    "    plt.plot(x_points,d1,color=\"b\")\n",
    "    plt.plot(x_points,d2,color=\"g\")\n",
    "    plt.plot(x_points,d3,color=\"r\")\n",
    "    plt.legend([\"sigma1\", \"sigma2\", \"mix\"])\n",
    "    plt.ylim(0,160)\n",
    "    plt.subplot(1,3,3)\n",
    "    plt.plot(x_points,d2,color=\"g\")\n",
    "    plt.plot(x_points,d3,color=\"r\")\n",
    "    plt.plot(x_points,d1,color=\"b\")\n",
    "    plt.legend([\"sigma2\", \"mix\", \"sigma1\"])\n",
    "    plt.ylim(0,80)\n",
    "    \n",
    "visualize_scale_mixture_components()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have all the components we need to construct a single bayesian network layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BayesianLinear(nn.Module):\n",
    "    def __init__(self, in_features, out_features):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        # Weight parameters\n",
    "        self.weight_mu = nn.Parameter(torch.Tensor(out_features, in_features).uniform_(-0.2, 0.2))\n",
    "        self.weight_rho = nn.Parameter(torch.Tensor(out_features, in_features).uniform_(-5,-4))\n",
    "        self.weight = Gaussian(self.weight_mu, self.weight_rho)\n",
    "        # Bias parameters\n",
    "        self.bias_mu = nn.Parameter(torch.Tensor(out_features).uniform_(-0.2, 0.2))\n",
    "        self.bias_rho = nn.Parameter(torch.Tensor(out_features).uniform_(-5,-4))\n",
    "        self.bias = Gaussian(self.bias_mu, self.bias_rho)\n",
    "        # Prior distributions\n",
    "        self.weight_prior = ScaleMixtureGaussian(PI, SIGMA_1, SIGMA_2)\n",
    "        self.bias_prior = ScaleMixtureGaussian(PI, SIGMA_1, SIGMA_2)\n",
    "        self.log_prior = 0\n",
    "        self.log_variational_posterior = 0\n",
    "\n",
    "    def forward(self, input, sample=False, calculate_log_probs=False):\n",
    "        if self.training or sample:\n",
    "            weight = self.weight.sample()\n",
    "            bias = self.bias.sample()\n",
    "        else:\n",
    "            weight = self.weight.mu\n",
    "            bias = self.bias.mu\n",
    "        if self.training or calculate_log_probs:\n",
    "            self.log_prior = self.weight_prior.log_prob(weight) + self.bias_prior.log_prob(bias)\n",
    "            self.log_variational_posterior = self.weight.log_prob(weight) + self.bias.log_prob(bias)\n",
    "        else:\n",
    "            self.log_prior, self.log_variational_posterior = 0, 0\n",
    "\n",
    "        return F.linear(input, weight, bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And can also construct the full 2-layer fully connected neural network model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BayesianNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.l1 = BayesianLinear(28*28, 400)\n",
    "        self.l2 = BayesianLinear(400, 400)\n",
    "        self.l3 = BayesianLinear(400, 10)\n",
    "    \n",
    "    def forward(self, x, sample=False):\n",
    "        x = x.view(-1, 28*28)\n",
    "        x = F.relu(self.l1(x, sample))\n",
    "        x = F.relu(self.l2(x, sample))\n",
    "        x = F.log_softmax(self.l3(x, sample), dim=1)\n",
    "        return x\n",
    "    \n",
    "    def log_prior(self):\n",
    "        return self.l1.log_prior \\\n",
    "               + self.l2.log_prior \\\n",
    "               + self.l3.log_prior\n",
    "    \n",
    "    def log_variational_posterior(self):\n",
    "        return self.l1.log_variational_posterior \\\n",
    "               + self.l2.log_variational_posterior \\\n",
    "               + self.l2.log_variational_posterior\n",
    "    \n",
    "    def sample_elbo(self, input, target, samples=SAMPLES):\n",
    "        outputs = torch.zeros(samples, BATCH_SIZE, CLASSES).to(DEVICE)\n",
    "        log_priors = torch.zeros(samples).to(DEVICE)\n",
    "        log_variational_posteriors = torch.zeros(samples).to(DEVICE)\n",
    "        for i in range(samples):\n",
    "            outputs[i] = self(input, sample=True)\n",
    "            log_priors[i] = self.log_prior()\n",
    "            log_variational_posteriors[i] = self.log_variational_posterior()\n",
    "        log_prior = log_priors.mean()\n",
    "        log_variational_posterior = log_variational_posteriors.mean()\n",
    "        negative_log_likelihood = F.nll_loss(outputs.mean(0), target, reduction='sum')\n",
    "        loss = (log_variational_posterior - log_prior)/NUM_BATCHES + negative_log_likelihood\n",
    "        return loss, log_prior, log_variational_posterior, negative_log_likelihood\n",
    "\n",
    "net = BayesianNetwork().to(DEVICE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tensorboard is a useful tool for keeping track of evolution of training and network sanity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_weight_histograms(epoch):\n",
    "    writer.add_histogram('histogram/w1_mu', net.l1.weight_mu,epoch)\n",
    "    writer.add_histogram('histogram/w1_rho', net.l1.weight_rho,epoch)\n",
    "    writer.add_histogram('histogram/w2_mu', net.l2.weight_mu,epoch)\n",
    "    writer.add_histogram('histogram/w2_rho', net.l2.weight_rho,epoch)\n",
    "    writer.add_histogram('histogram/w3_mu', net.l3.weight_mu,epoch)\n",
    "    writer.add_histogram('histogram/w3_rho', net.l3.weight_rho,epoch)\n",
    "    writer.add_histogram('histogram/b1_mu', net.l1.bias_mu,epoch)\n",
    "    writer.add_histogram('histogram/b1_rho', net.l1.bias_rho,epoch)\n",
    "    writer.add_histogram('histogram/b2_mu', net.l2.bias_mu,epoch)\n",
    "    writer.add_histogram('histogram/b2_rho', net.l2.bias_rho,epoch)\n",
    "    writer.add_histogram('histogram/b3_mu', net.l3.bias_mu,epoch)\n",
    "    writer.add_histogram('histogram/b3_rho', net.l3.bias_rho,epoch)\n",
    "\n",
    "def write_loss_scalars(epoch, batch_idx, loss, log_prior, log_variational_posterior, negative_log_likelihood):\n",
    "    writer.add_scalar('logs/loss', loss, epoch*NUM_BATCHES+batch_idx)\n",
    "    writer.add_scalar('logs/complexity_cost', log_variational_posterior-log_prior, epoch*NUM_BATCHES+batch_idx)\n",
    "    writer.add_scalar('logs/log_prior', log_prior, epoch*NUM_BATCHES+batch_idx)\n",
    "    writer.add_scalar('logs/log_variational_posterior', log_variational_posterior, epoch*NUM_BATCHES+batch_idx)\n",
    "    writer.add_scalar('logs/negative_log_likelihood', negative_log_likelihood, epoch*NUM_BATCHES+batch_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "(data, target) = next(enumerate(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(net, optimizer, epoch):\n",
    "    net.train()\n",
    "    if epoch == 0: # write initial distributions\n",
    "        write_weight_histograms(epoch)\n",
    "    for batch_idx, (data, target) in enumerate(tqdm(train_loader)):\n",
    "        data, target = data.to(DEVICE), target.to(DEVICE)\n",
    "        net.zero_grad()\n",
    "        loss, log_prior, log_variational_posterior, negative_log_likelihood = net.sample_elbo(data, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        write_loss_scalars(epoch, batch_idx, loss, log_prior, log_variational_posterior, negative_log_likelihood)\n",
    "    write_weight_histograms(epoch+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 600/600 [05:36<00:00,  2.46it/s]\n",
      " 43%|████▎     | 256/600 [02:21<03:21,  1.71it/s]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-cec21c699ccf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTRAIN_EPOCHS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-15-a25aa0865751>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(net, optimizer, epoch)\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_prior\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_variational_posterior\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnegative_log_likelihood\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample_elbo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-350ee8ccc96b>\u001b[0m in \u001b[0;36msample_elbo\u001b[0;34m(self, input, target, samples)\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0mlog_variational_posteriors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m             \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m             \u001b[0mlog_priors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_prior\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mlog_variational_posteriors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_variational_posterior\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-350ee8ccc96b>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, sample)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-9d1bec9f705a>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, sample, calculate_log_probs)\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcalculate_log_probs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m             \u001b[0mweight\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m             \u001b[0mbias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-e450442d06bb>\u001b[0m in \u001b[0;36msample\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m         \u001b[0mepsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrho\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmu\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msigma\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/lib/python3.7/site-packages/torch/distributions/normal.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, sample_shape)\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extended_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrsample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "optimizer = optim.Adam(net.parameters())\n",
    "for epoch in range(TRAIN_EPOCHS):\n",
    "    train(net, optimizer, epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n",
    "\n",
    "## Model ensemble"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Having trained distributions on the weights of our model, we now effectively have an infinite ensemble of neural networks. We can leverage this by combining the outputs from different samples of our model weights:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-79149eeef3cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Ensemble Accuracy: {}/{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorrect\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTEST_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mtest_ensemble\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-21-79149eeef3cc>\u001b[0m in \u001b[0;36mtest_ensemble\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTEST_SAMPLES\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTEST_BATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCLASSES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTEST_SAMPLES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m                 \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m             \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTEST_SAMPLES\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-350ee8ccc96b>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, sample)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ml3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-9d1bec9f705a>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, sample, calculate_log_probs)\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcalculate_log_probs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m             \u001b[0mweight\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m             \u001b[0mbias\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-e450442d06bb>\u001b[0m in \u001b[0;36msample\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m         \u001b[0mepsilon\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrho\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDEVICE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmu\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msigma\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda/lib/python3.7/site-packages/torch/distributions/normal.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, sample_shape)\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extended_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrsample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def test_ensemble():\n",
    "    net.eval()\n",
    "    correct = 0\n",
    "    corrects = np.zeros(TEST_SAMPLES+1, dtype=int)\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(DEVICE), target.to(DEVICE)\n",
    "            outputs = torch.zeros(TEST_SAMPLES+1, TEST_BATCH_SIZE, CLASSES).to(DEVICE)\n",
    "            for i in range(TEST_SAMPLES):\n",
    "                outputs[i] = net(data, sample=True)\n",
    "            outputs[TEST_SAMPLES] = net(data, sample=False)\n",
    "            output = outputs.mean(0)\n",
    "            preds = preds = outputs.max(2, keepdim=True)[1]\n",
    "            pred = output.max(1, keepdim=True)[1] # index of max log-probability\n",
    "            corrects += preds.eq(target.view_as(pred)).sum(dim=1).squeeze().cpu().numpy()\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "    for index, num in enumerate(corrects):\n",
    "        if index < TEST_SAMPLES:\n",
    "            print('Component {} Accuracy: {}/{}'.format(index, num, TEST_SIZE))\n",
    "        else:\n",
    "            print('Posterior Mean Accuracy: {}/{}'.format(num, TEST_SIZE))\n",
    "    print('Ensemble Accuracy: {}/{}'.format(correct, TEST_SIZE))\n",
    "\n",
    "test_ensemble()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Due to the choice of a gaussian distribution for our variational posterior, the mean weights are also the median and mode weights. We can see from these results that combining the outputs of samples of our model provides a better test accuracy than any individual sample (including that from the mean weights)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model uncertainty\n",
    "\n",
    "#### In-domain uncertainty "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show(img):\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9, 2, 1, 1, 6])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAABoCAYAAADo66t9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAeKUlEQVR4nO2de3AW1fnHPwilpUVqoYQgUNpIQYpctCCgKCVIgIZIFOiM7dAO0KEXpynSgSkwvQwVLI4y1ZlKidSqnV6kIKggrUMQQyuKXJSi2GIRCApJpVwEIZCwvz+Y7+7JySbkRt53fzyff/Lmfffd9+zZ3bPf89xOiyAIAgzDMIzEcUWqG2AYhmE0DBvADcMwEooN4IZhGAnFBnDDMIyEYgO4YRhGQrEB3DAMI6E0agAvLi5m9OjRjBo1isLCwqZqk2EYhlEHGjyAV1ZWMn/+fJYtW8batWtZs2YN77zzTlO2zTAMw6iFVg394s6dO+nevTvdunUDIDc3l6KiInr06FHjd/773/+yf//+hv6kYRjGZcnAgQNj32/wAF5aWkpmZmb4f6dOndi5c2et39m/fz+DBg1q6E8ahmFcltSUMN9gE0rcDlu0aNHQ3RmGYRj1pMEDeGZmJocPHw7/Ly0tJSMjo0kaZRiGYVycBptQ+vbty759+ygpKaFTp06sXbuWBx98sCnbZjQBV155JQA33ngjAEVFRRf9zg033ADAyZMnAfj3v/99iVp3adBM0J0ljhw5EoCCggIAXn/9dYDQDOg64Nu2bQvAZz7zGQDOnTsHQFZWVrjNHXfccUnafqno2LEjANOnTwfg+PHjAJw+fbratvpM/deyZUsAWrduDUBZWVm47caNGwE4e/bsJWh1zVxxRaQ9z58/D1S3ANSlTt/QoUMB+OQnPxm+p+PUcYuPf/zjwAVfniguLq5Ps5ucBg/grVq14qc//Snf/va3qaysZMKECXzxi19syrYZhmEYtdDgARxg+PDhDB8+vKnaYhiGYdSDFs1ZD3zr1q0WhXIJ+MQnPgHAjBkzALjrrrvCz2QG0BT6o48+AqB9+/Y17u/MmTNANL2urKwE4KWXXgq3WbZsGQB//etfG38ATYym15paA/z9738H4Oabb479zokTJ8LXmk63anVB36jP3Gl2Xl4eAGvWrGmqZl9Svv/97wOwePFiAP73v/8BcOjQIaCqeei9994DItNZ7969gei6WL9+fbitIs9+//vfX7K2x1FbwETckCZTYnZ2NgDXX389AGPHjgXgX//6V7Xvy5TWoUMHAD744AMA2rRpE24rM4uug2effRaAAwcO1Ot4LkaTR6EYhmEYqaVRJhQjtSxatAiIHFNSGa5jSq+luKQeTp06BVR1BskRJcWpz+S8GTduXLjt+PHjAdi8eTMAt956a9McVBPgKm/Rv39/IOoHqalPfepTQFWH1ZEjRwCoqKgAIrXnJqlde+21QHIUuGZg+/btA6JZlZASh+i8S3m2a9cOiGYpV199dbjt22+/fWkafBFcRRrntIbovgBC/5zOsxT38uXLgej6ACgvLweiGZi21fHr/oCoXz/3uc8BhIEc2mbOnDnhtu+//369jrEumAI3DMNIKKbAE4iUxezZswHCeHyp6jh18rGPfQyI7Jj6626r11IeQtsqrBAiBXfTTTcB8NxzzwGRbTjdkD1TyluqUmpTqgsilaaZh/uZUAmJpCA1rRA42byPHj0KRP0B8OGHHwJw1VVXAdE1pBmee83885//vJTNrhHXBu4rb9n7XT+PSngoJFTnWCGRbjhgfn4+EN1Xmpnqd3bt2hVuO2bMGCAKQ5VKlyL/xS9+EW47bdq0eh1jXTAFbhiGkVBMgScQPdX1tJfNV6rCrVEjpLS0rey7sgFDFM0iO7HUqdS2FClECqi0tBSIbOCf/exngUjpppJOnTpVe08KTGpKx+jawNU36itt60aqJC3rWApUtl4dm86tZm8QKU71iZSoFK2rflNlA49T4JoVde3aFYB333033EYzMKHj1TXyn//8J/xs7969QGQ3l09EKvuWW24Jt5VdW/eO/sr35N6LkydPBqKInZps9/XBFLhhGEZCsQHcMAwjoZgJJYF8+tOfBiLnmswAmq498sgj4bZaKWnbtm1ANB3WNFMOK4iSD2Qe0FS6c+fOQJTgAZFjU84vhSfKOZYOJpTrrruu2nsyoai9MiG4YXXqT01xZUpwnZkyFSUFTdPldJQJQcd4zTXXhNsq+Uv94NfCkYkBInNTcxMXKqowT51L1xkvB7zMgDqnel8OW4iS04YNGwZE5hDtz92vTIgyRep+UD0V95oZMGAAEJlQmiKH0hS4YRhGQjEFnkCkIqSC/bTiuXPnhq9VWU6KQ8pTVeRGjBhRbf9vvfUWEKVQS1Wokh/AvffeC0RhaVJrUi1btmyp/4E1MW5yhmYT6jOlxasv3TA6OXGlkNS/rhPXdfolASnWkpISIDrHOsaJEyeG2yrksE+fPkBUQkGzOHcmJqXpJrekCrVX5zjO6e4nqUmtu+dfSU0vvPACEM0ytK3r8NR+5QyVOpcz00UVQZsSU+CGYRgJxRR4HZB6de1uvv0qLulDYUh79uxpdBukdFzUHldpADz55JPha6W8C4WCSXnPnz8//ExhciqGpW2VlKC0Y4gUuK9kZOdLB9zCaeorKW+pKvkTtm/fHm6rY1Dopc6pW8xKSjYp7N69G4jqout/HZsUOUSzp6VLlwLRsR48eBCI+gXi64mnii5dugCRX8e/LyCyWfsFy+QbgUjJy1+g+0Ahg24pAdnOpcCl3rUPN5RR4Yi6l5uihropcMMwjIRy2Slw2az011XVeoJrlY5169YBdbN3xqVb33nnnUBUdKoxuE99oba75S0hOo44Jk2aVOV/twyo1JRmHG+88QYQRaG4qfQ1kU6LesiGD5HCUp8psUOKaciQIeG2fpKP/rrRB7KTJwUpTl3Lilhy1bTQcUrB6vhlW3YjT2Trjbv+mws/YUvHqmgaiNS0n0ov3CgkHYu+L8Xsl6WAqB/1HfWZG9Ui1I/9+vUDLpTXbiymwA3DMBLKZafARVwcqVJkBw8eDESq9+GHH77o/hQ7PXr06PA9N8a6sahsZRxSBFIXrgJ3y8VC1UUZoOqCDIrhlq3uq1/9KgAvvvgiEClyiNS49i9VFpfGnypk34bq6fFS4E8//XSN35dK80uvQrxPIp2R8pY6VT/oGndnF1ovVDMRzfDi1op0bcepQtetjlEq2C0ToWORPVv3TFy0iO/X0Xd1j7v+L/2W9qe+irPD6xr8whe+AJgCNwzDuKyxAdwwDCOhXHYmFE3/NJ1xQ83k9FKokRxyq1atAqo6rjRVUpU3f/USiMKumoI4x6SfwKMkBdeMoamytu3VqxcAv/zlL4GqKdRCIWZadaZ79+5AVGcZIkev+kQhUbU5UJsbt2Kg+sYP//zTn/5U7XtySGm6LZOSixtSmATkoNb14Duk3f937NhR5TNd69qH67BMBxOKqhDKyeqbDd1tVC7Cr7jomoVkitGxKYzWNxdC9dV75PDX9eb2j1737Nmz3sdYE6bADcMwEsplo8D9p6ccHG4KsZ6mcmxoBRKpV/fJrvcUsK9kBzcsy1/ZpjHEOTH9OuD6PVdNLViwAIicLDk5OUCUZu4WfNLxSnlLpT/11FNAfJKOn+TkhlilGlclq0/8cyIHrYvW+dQsww85g3hVns7IIefXQ9ffuBBRKVo5L6VMXQUa5+BtbuSI1TWohDTXgaiZsbbRdaD2u+fYT5DTZ3JMuuGJ6iPNUvTbKnamUhYQjRluiYfGYgrcMAwjoSRSgfsrWUgZ+6oCag4F++53vwtE9m6Inqay+UqJa5u4p7RUiWxqrg3cD2dqTAEk2dZc1AY/0cR96ruFrdzPdExf+tKXqu1XJWel+tUvLurjuDID7vuQHipNaIYgFRmXgKKV21WYy/c1QNU+TgIq7+vfM1LXcedYilPHr23c1dXjwnGbG4WE6h7ULFi2a4Bnn322yrbqB81IXLXuhwbqWvETnCA6fvWNVrDX2rBu/+g+8BPvGoMpcMMwjISS9go8bt04P5KgPupPhZoUqeF63PWEVRqs7JyKtHCL+Mte7NtHXTu5bLCKZlGCREOoLZFHyqOoqAiI1qeEKBJG/SDFpWONSzaS8pBKl+Jwt5UClV3ctwl//vOfD1+75TdTha4ZHVttbVKf+TO7JKOSAX5auK7RON+FrhHNHGXfjYvySCW6PhUlI8XszpxUrEvJer7N3x1DdP/XtI6s6wPwZ2da/EL96n6u2Z6bYNZY0utMGIZhGHUm7RV4nPrxiwz5Ka9xNtepU6cCUQymokYUvw3R01I2KhWul9p2n9KK85SdvLYVppVe3xgFHlccR/Y8KcYnnngCiFLg3XYKf7mwuEgZHYNv83OVx+9+9zug5vKx7mwlHRS4H320a9euGrd9/vnnAZg9ezaQfoqzIeg60F+/FIJi3l381HQpyHSIwHGvW80eNBvWte3GYMtu7ytmv7yw+56O0/etufv1l93TyvXah3vtqM26BnX/1qVIXE0k/8o0DMO4TLEB3DAMI6GknQnFn6766xJCZMqoKYTJrZ2tmtwyi2h1HE1f3JAgmVPkFNRvx6VNy0yjaaX+d0MF1b6bb745tp31wZ3i+u3SupRxtZ013dP0rS4OOT9EUP+7FfheffXV2O/IkZRuZgffdOSulOKjqos63jgHX9LWxPSvTz/0VNeQi+4V3Tu1VfBrblwTnW++1HXrrnjjrygvk4nOsWvG0DXsH6/uJddEq9/WezLVxJlxFWqodiqQQmaXhpBed5lhGIZRZ1KqwOPC/WpS1XHKUaF1CllToSY36UVPYYVAyRmohBtXXUmNqw1K6NE2x44dC7f1V3iRonHXCPRTcJV2/+abb8YeY224Tkw/5V/qwV2BRkhp+CqyNiXuKxr9ddvgf98vN+AqpFThFhPz62C7ySg+rkML4lPpk6bAhRSoZnT6P272ptC7rl27AtE9kw7rYLrXoo5BClefueuW6h6UA1EhsvquO2PUWKT7SzMQve9eH7qvFOigmX1ZWRlQdTzTdaTfUrE1U+CGYRiXIRdV4IcOHWL27Nl88MEHXHHFFXzta1/jW9/6FseOHeOee+7hvffeo0uXLvzqV7+qd4B6nJ1I69tJ/eqJ6a6uoSeiVraQupIqdu1ZetqpbfqunqKufVshVlK4so8pCcI9Pn1PykVPXrfQjb/+oBuyWF9cFeirX6XvxpWGrancQFx6uP8d/WZcAoIUht8+7be2xKPmwi2ToL5RO2sr6emvFh53nSatnKzQNSj79tixY4FoBXqX7du3A3DjjTcCUVhtOvg33HtA97uuU51b3RfuNjXNrtwZqvat/WnGIUUedy/qvtc9rzU4pcyhemKQxozGcNEz0bJlS3784x+zbt06nnrqKf74xz/yzjvvUFhYyNChQ3nhhRcYOnQohYWFjW6MYRiGUXcuqsAzMjJCW03btm3JysqitLSUoqKicEXz/Px8Jk+ezKxZsxrckNtuuw2IIkj0pJSSi7OXaxvZt/REcxc0kCKUfVtPQakI9ymo39BTVE9tpY27CwT4+E9XiNR+XCJMfXETF3xFqPRdN4U+7nsQ9UdtiUe+PTuu3bIv668/u2gKddFYXnvttfC1/ANSVfUp6elGKolUrsLeGIYPHw5EMxIp8MmTJ1fbVslOspfffffdAOzcuTPcZtu2bZeusbXgXm9+kSjNFN12ahzxi1nFFajSTFzXvb82qDsD8X1hKqC1d+9eICpJ7H7/7bffBqoWvmso9ZoLHTx4kN27d9O/f3+OHDkSDmgZGRlVVqsxDMMwLj11HsBPnTpFQUEBc+fOTQt1ZRiGcblTpzDCc+fOUVBQQF5eXriiS4cOHSgrKyMjI4OysrLYWgoXQ/sCmDZtGhBNL+Q4lHnEnbb469lpyq9t3YQTTW381XU0nXFNHnJkyAQjh6pqZrv79R05Mru4zi2FNckU4zv+6oNbr9k3oegYtJKOW6uhPg4n39Gp/cY58Xr06AFEtcPVZzo36eDkKy4uDl9PmTIFiPrmhhtuuOj341ZrEelQB7s++DU7VCFTIWxx9cBlQpBJYvDgwUB6rLrknj85F/VX960bGjlw4ECg+tqgvsMeomvYDzH0nfrua/WVTHMKW3ZDLjXmKCDjy1/+MgArVqyo83H7XPTuDoKAefPmkZWVFd4EANnZ2axevRqA1atXM3LkyAY3wjAMw6g/F1Xg27Zt45lnnqFnz56MHz8egJkzZzJ9+nRmzJjBihUr6Ny5Mw899FC9f3zLli3h6yFDhgDQt29foHr6uetIk9KW3V1/5Wx0lbKUh5weSvaRQnQdCXoa6ykqJ4hWaJGjFSKnh+8EdNupsCs9jd2Qovri7tdXhFJEOka3AmGcenSpLaHHX3PTRdeC+ub666+vsj83nDJVvPzyy+FrKUz1Y11mQ/6KNC7pEEpXH/xyCFKDtTlj/drxUuJNudZrQ3ETqaS8u3TpAkT3mUoiQHRPKxnPnyG659hfC1P3gWbSbpip/IB+8t9zzz0HwLJly8Jtly9fXqXtmr02houeiYEDB1aJp3RR+VLDMAyj+Unpo9RNTZ8/f36Vz+Qold1NyhngpptuAqIU+n79+gGRbcl9mvr2XKl1BdqvX78+3FZ1oOPsgRCtqwdRuJDWGpRac1et8dddVLhfQ3Dt0H4xIdm+pZhcVSW1pOP31aT7f02hhXEKXH2vWcrEiROrfDcd7KT79+8PX/srlasPs7KygCjsy8UvBOZysZlNuuKv3VpbSQC/eJPOaVMox8aievQuGjN0Tt069Cpq5yfe6Zp3xyKVgdD95M9E3OtB95WKgcmSoLwYN/RYCr6m8aUhJGseaBiGYYSk3phVA3paaZ1H/QV45JFHUtKm22+/PSW/C1Xtbr6Klr1Zdk13Wz9awv8/bq1RP91ef92V2JWg4M8q/KSHdMG3a+r/2hS4IqE023BzHZKqwP208NrUoK4jP7HLjXJKJzRmaFbo+pwUJadz6K/76trEta2OW/2ge8dN+vHLLmg/WqlKs/pLhSlwwzCMhJK2Ctyoiqt6pKJkx1u8eDFAGMrpqt+4GG6orrahurKXytQ+3IidjRs3ArBmzRoAfvazn1XZ1o0Eam7ibPmrVq0C4Otf/3qVbYYNGwZU9YUI3z7s9k9c+dUkoHh9v7RpHFK0fjRSOpSTdfFnCLoGdW6h+qxBkVr6jmZiEEVWCcWVa1vXB6X9qE8UeXbLLbcAVRV4beUrGoopcMMwjIRiA7hhGEZCMRNKQnCdLJoialooh4xCGpUmDVEoVU1T5bgkFb2nqbPCId1yCUqE0W/6bVNCQyqIm6o+88wzAHzzm98Eor6bMGECAD//+c+r7cdfR9TdX1KrEcppp/C22ipkykykcyrnXWNKQlwKdF58c6EbeqwwUpn2tK3uFTfkVKYzVUbVceu6ck0ovqNTf92KqH47m9KUYgrcMAwjoZgCTwhuWrhC+BQCplC+2laZuVTI+aMEJqkTtxZ3c6PZhhsyuW7dOiBSlf76p3GoHrbKO7jOO3fd1SShflBxp9qOX+dU6lXn1lWr6YTvdFeyHUTKW8W7dNy6d9wQURWv0zaa4Wq/7opffvkOXVeaMbshh5q1mQI3DMMwTIEnBbfwl57ufoJBKpA6kdKQEnFVSnNTU+gkwIEDB4Ao5VnlF1SeAaLZjhSdlKdbHkDp1klDszYdU219JfwyqFqFKd3wFe28efPC11otbMyYMUC0cv27774LVPUFqG+UHq9EOSUGub4ghRhKicsn9Otf/xqI95U05f1qCtwwDCOhmAJPCEoQgGi1cKkpP+Ekbv3M2lahryvuPrRf2RTXrl0LRCVHX3nllUb/XkOpzbb46KOPAtHCIX/+85+Bqj4GoTVfdUzurGLTpk1N09hm5sknnwSiJBfZxGvDLeIGUSG4dMNXtq7Pwi+WJ/u47N1S0hAlrPmRW4pccpOCNKP7xz/+ATT/zNMUuGEYRkKxAdwwDCOhtAiaMjH/ImzdupVBgwY1188ZhmH8v6CmYdoUuGEYRkKxAdwwDCOh2ABuGIaRUJrVBm4YhmE0HabADcMwEooN4IZhGAnFBnDDMIyEYgO4YRhGQrEB3DAMI6HYAG4YhpFQbAA3DMNIKM02gBcXFzN69GhGjRpFYWFhc/1snTl06BCTJ09m7Nix5Obm8sQTTwBw7NgxpkyZQk5ODlOmTAkLt6cLlZWV5Ofn853vfAeAkpISJk2aRE5ODjNmzAgXfUgXTpw4QUFBAWPGjGHs2LHs2LEjrfv48ccfJzc3l3HjxjFz5kzKy8vTqo/nzJnD0KFDGTduXPheTf0ZBAH33nsvo0aNIi8vjzfffDMt2rto0SLGjBlDXl4ed999d7iEG8DSpUsZNWoUo0ePTkkJ37j2it/+9rf06tUrXI4tJf0bNAMVFRXByJEjgwMHDgTl5eVBXl5esGfPnub46TpTWloa7Nq1KwiCIPjwww+DnJycYM+ePcGiRYuCpUuXBkEQBEuXLg3uv//+VDazGo899lgwc+bMYPr06UEQBEFBQUGwZs2aIAiC4Cc/+Unwhz/8IZXNq8bs2bOD5cuXB0EQBOXl5cHx48fTto8PHz4cjBgxIjh9+nQQBBf6duXKlWnVx1u2bAl27doV5Obmhu/V1J8bN24Mpk2bFpw/fz7YsWNHMHHixLRo76ZNm4Jz584FQRAE999/f9jePXv2BHl5eUF5eXlw4MCBYOTIkUFFRUXK2xsEQfD+++8HU6dODb7yla8ER44cCYIgNf3bLAp8586ddO/enW7dutG6dWtyc3MpKipqjp+uMxkZGfTp0weAtm3bkpWVRWlpKUVFReTn5wOQn5/P+vXrU9nMKhw+fJiNGzcyceJE4IICeOWVVxg9ejQAd9xxR1r188mTJ3nttdfC9rZu3Zp27dqldR9XVlZy5swZKioqOHPmDB07dkyrPh40aFC44ISoqT/1fosWLRgwYAAnTpygrKws5e0dNmxYuAjJgAEDOHz4cNje3NxcWrduTbdu3ejevTs7d+5MeXsB7rvvPmbNmlVlkZNU9G+zDOClpaVkZmaG/3fq1InS0tLm+OkGcfDgQXbv3k3//v05cuQIGRkZwIVB3l29OtUsXLiQWbNmhSuHHD16lHbt2oU3Q2ZmZlr1c0lJCe3bt2fOnDnk5+czb948Pvroo7Tt406dOjF16lRGjBjBsGHDaNu2LX369EnrPgZq7E//PkzHtq9cuZJbb70VSN9xo6ioiIyMDK699toq76eif5tlAA9iyq00xRJfl4JTp05RUFDA3Llzadu2baqbUyMvvvgi7du357rrrqt1u3Tq54qKCt566y3uuusuVq9eTZs2bdLSHyKOHz9OUVERRUVFbNq0idOnT1NcXFxtu3Tq49pI9/twyZIltGzZkttvvx1Iz/aePn2a3/zmN/zwhz+s9lkq2tssa2JmZmaG0yK48KSSQkgnzp07R0FBAXl5eeTk5ADQoUMHysrKyMjIoKysrMqK1Klk+/btbNiwgeLiYsrLyzl58iQLFizgxIkTVFRU0KpVKw4fPpxW/ZyZmUlmZib9+/cHLqwQXlhYmLZ9/PLLL9O1a9ewPTk5OezYsSOt+xhqvmb9+zCd2r5q1So2btzI448/Hg566ThuHDhwgIMHDzJ+/HjgQh/eeeed/OUvf0lJ/zaLAu/bty/79u2jpKSEs2fPsnbtWrKzs5vjp+tMEATMmzePrKwspkyZEr6fnZ3N6tWrAVi9ejUjR45MVROr8KMf/Yji4mI2bNjA4sWLGTJkCA8++CCDBw/mb3/7G3Dhpkinfu7YsSOZmZns3bsXgM2bN3PNNdekbR9fffXVvPHGG5w+fZogCNi8eTM9evRI6z6Gmq9ZvR8EAa+//jpXXnllygdEuBCh9uijj7JkyRLatGkTvp+dnc3atWs5e/YsJSUl7Nu3j379+qWwpdCrVy82b97Mhg0b2LBhA5mZmTz99NN07NgxJf3bbOVkX3rpJRYuXEhlZSUTJkzge9/7XnP8bJ3ZunUr3/jGN+jZs2doU545cyb9+vVjxowZHDp0iM6dO/PQQw9x1VVXpbi1VXn11Vd57LHHWLp0KSUlJdxzzz0cP36c3r1788ADD9C6detUNzFk9+7dzJs3j3PnztGtWzfuu+8+zp8/n7Z9/PDDD/P888/TqlUrevfuzYIFCygtLU2bPp45cyZbtmzh6NGjdOjQgR/84Afcdtttsf0ZBAHz589n06ZNtGnThoULF9K3b9+Ut7ewsJCzZ8+G57x///7hKvJLlixh5cqVtGzZkrlz5zJ8+PCUt3fSpEnh59nZ2axYsYL27dunpH+tHrhhGEZCsUxMwzCMhGIDuGEYRkKxAdwwDCOh2ABuGIaRUGwANwzDSCg2gBuGYSQUG8ANwzASyv8BFOWOyv66KLEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fmnist_sample, fmnist_target = iter(test_loader).next()\n",
    "fmnist_sample = fmnist_sample.to(DEVICE)\n",
    "print(fmnist_target)\n",
    "sns.set_style(\"dark\")\n",
    "show(make_grid(fmnist_sample.cpu()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instead of just combining the outputs of samples from our model, we can instead graph a histogram of each sample’s individual prediction. By doing so we can now visualize our model’s uncertainty on the following five entries from the FMNIST test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAEKCAYAAAC8B0kLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dfVxUZd4/8M/M6KYi8TCKsOSutz1phEumxGwG6yCk8iD6yjBbN4mNNR/R1lXoJi2NvavNW802m1vzxofVW1EnH8JILC3LJ0JRs1IKqrmDkRkQZBKY4fr9wc9zR8g4jpxhnPm8Xy9er5mLw3V9v2fmnPlynTPnKIQQAkRERETUZZRdHQARERGRt2NBRkRERNTFWJARERERdTEWZERERERdjAUZERERURdjQUZERETUxWQryLKysqDRaJCYmCi11dbWIi0tDfHx8UhLS8OlS5cAAEIILF26FHFxcUhKSsLZs2flCouIiIjI7chWkE2YMAFr1qxp06bT6aDRaFBYWAiNRgOdTgcAOHToEMrLy1FYWIglS5Zg8eLFcoVFRERE5HZkK8iGDx8OPz+/Nm1FRUVISUkBAKSkpGD//v1t2hUKBSIiIlBXVwej0ShXaERERERupZsrBzOZTAgKCgIABAUFwWw2AwCqqqoQHBwsLRccHIyqqipp2Y4IIeAp9xlQKMBc3Iyn5AEwF3fkKXkAzMVduSoXpVIh/yAu0NIi/8qyt65cWpB15Fp3b1Iorv8CW60tqK21yBGSy/n792IubsZT8gCYizvylDwA5uKuXJFL376+GLBwr6xjuEr5fyTg4sV6Wcfo29e3w9+59FuWarVaOhRpNBoRGBgIoHVGrLKyUlqusrLyurNjRERERJ7CpQWZVquFXq8HAOj1esTGxrZpF0Lg5MmT8PX1ZUFGREREXkO2Q5bz5s3DsWPHUFNTg+joaMyaNQsZGRnIzMxEfn4+QkJCsGLFCgBATEwMDh48iLi4OPTs2RO5ublyhUVERETkdmQryJYtW3bN9ry8vHZtCoUCixYtkisUIiIiIrfGK/UTERERdTGHCrLi4mKH2oiIiIjoxjlUkC1dutShNiIiIiK6cXbPISspKUFJSQnMZjPWrVsntV++fBk2m0324IiIiIi8gd2CrLm5GRaLBTabDQ0NDVJ77969sXLlStmDIyIiIvIGdguyyMhIREZGYvz48QgNDXVVTERERERexaHLXjQ1NSEnJwcGgwFWq1VqX79+vWyBEREREXkLhwqyOXPmYNKkSZg4cSKUSl4pg4iIiKgzOVSQdevWDZMnT5Y7FiIiIiKv5NB018iRI7Fp0yYYjUbU1tZKP0RERER08xyaIdu5cycAYO3atVKbQqFAUVGRU4NqtVr4+PhAqVRCpVJhx44dqK2txdy5c2EwGBAaGorly5fDz8/Pqf6JiIiIbiUOFWQHDhzo9IHz8vIQGBgoPdfpdNBoNMjIyIBOp4NOp8P8+fM7fVwiIiIid+NQQabX66/ZnpKS0mmBFBUVYcOGDVK/U6ZMYUFGREREXsGhguz06dPS48bGRnz22WcICwu7qYIsPT0dCoUCqampSE1NhclkQlBQEAAgKCgIZrP5un2oVAr4+/dyOgZ3olIpmYub8ZQ8AObijjwlD4C5uCtPysVVunJ9OVSQ5eTktHleX19/U7NXmzdvRr9+/WAymZCWloaBAwc61Y/NJlBba3E6Dnfi79+LubgZT8kDYC7uyFPyAJiLu3JFLn37+srav6t15fpy6qJiPXr0QEVFhdMB9evXDwCgVqsRFxeH0tJSqNVqGI1GAIDRaGxzfhkRERGRJ3NohmzatGnS45aWFpSVlWHMmDFODWixWNDS0oLevXvDYrHg8OHDmD59OrRaLfR6PTIyMqDX6xEbG+tU/0RERES3GocKsqefflp6rFKpEBoaiuDgYKcGNJlMmDFjBgDAZrMhMTER0dHRCA8PR2ZmJvLz8xESEoIVK1Y41T8RERHRrcahgiwyMhLV1dXSyf0DBgxwesD+/ftj165d7doDAgKQl5fndL9EREREtyqHziF77733MHHiROzbtw8FBQXSYyIiIiK6eQ7NkK1evRr5+flQq9UAALPZjKlTp2L06NGyBkdERETkDRyaIRNCSMUYAPj7+0MIIVtQRERERN7EoRmyESNGID09HQkJCQBaD2FGR0fLGhgRERGRt7BbkFVUVKC6uhoLFixAYWEhiouLIYRAREQEkpOTXRUjERERkUeze8gyNzcXPj4+AID4+HhkZWUhOzsbMTExyM3NdUmARERERJ7ObkFmMBgwaNCgdu3h4eEwGAyyBUVERETkTewWZI2NjR3+7sqVK50eDBEREZE3sluQhYeHY+vWre3at23bhrCwMNmCIiIiIvImdk/qz87OxsyZM7F7926pADtz5gyam5uxatUqlwRIRERE5OnsFmR9+vTBli1bcOTIEZw/fx4AEBMTA41G45LgiIiIiLyBQ9chi4qKQlRUlNyxEBEREXklh67UT0RERETyUQjeA4mIiIioS3GGjIiIiKiLsSAjIiIi6mIsyIiIiIi6mGwFWVZWFjQaDRITE6W22tpapKWlIT4+Hmlpabh06RIAQAiBpUuXIi4uDklJSTh79qxcYRERERG5HdkKsgkTJmDNmjVt2nQ6HTQaDQoLC6HRaKDT6QAAhw4dQnl5OQoLC7FkyRIsXrxYrrCIiIiI3I5sBdnw4cPh5+fXpq2oqAgpKSkAgJSUFOzfv79Nu0KhQEREBOrq6mA0GuUKjYiIiMitOHRh2M5iMpkQFBQEAAgKCoLZbAYAVFVVITg4WFouODgYVVVV0rIdEULAUy7aoVCAudwgpVIh/yAu0NIi/8ri+8v9eEoeAHNxV56Si6fkAdj/3HJpQdaRa10KTaG4/oet1dqC2lqLHCG5nL9/L+ZyA/r29cWAhXtlHcNVyv8jARcv1ss6Bt9f7sdT8gCYi7vylFw8JQ+g9bOrIy79lqVarZYORRqNRgQGBgJonRGrrKyUlqusrLzu7BgRERGRp3BpQabVaqHX6wEAer0esbGxbdqFEDh58iR8fX1ZkBEREZHXkO2Q5bx583Ds2DHU1NQgOjoas2bNQkZGBjIzM5Gfn4+QkBCsWLECABATE4ODBw8iLi4OPXv2RG5urlxhEREREbkd2QqyZcuWXbM9Ly+vXZtCocCiRYvkCoWIiIjIrfFK/URERERdzKGCrLi42KE2IiIiIrpxDhVkS5cudaiNiIiIiG6c3XPISkpKUFJSArPZjHXr1kntly9fhs1mkz04IiIiIm9gtyBrbm6GxWKBzWZDQ0OD1N67d2+sXLlS9uCIiIiIvIHdgiwyMhKRkZEYP348QkNDXRUTERERkVdx6LIXTU1NyMnJgcFggNVqldrXr18vW2BERERE3sKhgmzOnDmYNGkSJk6cCKWSV8ogIiIi6kwOFWTdunXD5MmT5Y6FiIiIyCs5NN01cuRIbNq0CUajEbW1tdIPEREREd08h2bIdu7cCQBYu3at1KZQKFBUVOTUoFqtFj4+PlAqlVCpVNixYwdqa2sxd+5cGAwGhIaGYvny5fDz83OqfyIiIqJbiUMF2YEDBzp94Ly8PAQGBkrPdTodNBoNMjIyoNPpoNPpMH/+/E4fl4iIiMjdOFSQ6fX6a7anpKR0WiBFRUXYsGGD1O+UKVNYkBEREZFXcKggO336tPS4sbERn332GcLCwm6qIEtPT4dCoUBqaipSU1NhMpkQFBQEAAgKCoLZbL5uHyqVAv7+vZyOwZ2oVErm4sXkXl+e9Jp4Si6ekgfAXNyVp+TiKXlcj0MFWU5OTpvn9fX1NzV7tXnzZvTr1w8mkwlpaWkYOHCgU/3YbAK1tRan43An/v69mMsN6NvXV9b+XU3u9cX3l/vxlDwA5uKuPCUXT8kDsP/Z5dRFxXr06IGKigqnA+rXrx8AQK1WIy4uDqWlpVCr1TAajQAAo9HY5vwyIiIiIk/m0AzZtGnTpMctLS0oKyvDmDFjnBrQYrGgpaUFvXv3hsViweHDhzF9+nRotVro9XpkZGRAr9cjNjbWqf6JiIiIbjUOFWRPP/209FilUiE0NBTBwcFODWgymTBjxgwAgM1mQ2JiIqKjoxEeHo7MzEzk5+cjJCQEK1ascKp/IiIioluNQwVZZGQkqqurpZP7BwwY4PSA/fv3x65du9q1BwQEIC8vz+l+iYiIiG5VDp1D9t5772HixInYt28fCgoKpMdEREREdPMcmiFbvXo18vPzoVarAQBmsxlTp07F6NGjZQ2OiIiIyBs4NEMmhJCKMQDw9/eHEEK2oIiIiIi8iUMzZCNGjEB6ejoSEhIAtB7CjI6OljUwIiIiIm9htyCrqKhAdXU1FixYgMLCQhQXF0MIgYiICCQnJ7sqRiIiIiKPZveQZW5uLnx8fAAA8fHxyMrKQnZ2NmJiYpCbm+uSAImIiIg8nd2CzGAwYNCgQe3aw8PDYTAYZAuKiIiIyJvYLcgaGxs7/N2VK1c6PRgiIiIib2S3IAsPD8fWrVvbtW/btg1hYWGyBUVERETkTeye1J+dnY2ZM2di9+7dUgF25swZNDc3Y9WqVS4JkIiIiMjT2S3I+vTpgy1btuDIkSM4f/48ACAmJgYajcYlwRERERF5A4euQxYVFYWoqCi5YyEiIiLySg5dqZ+IiIiI5KMQvAcSERERUZfiDBkRERFRF2NBRkRERNTFWJARERERdTHZCrKsrCxoNBokJiZKbbW1tUhLS0N8fDzS0tJw6dIlAIAQAkuXLkVcXBySkpJw9uxZucIiIiIicjuyFWQTJkzAmjVr2rTpdDpoNBoUFhZCo9FAp9MBAA4dOoTy8nIUFhZiyZIlWLx4sVxhEREREbkd2Qqy4cOHw8/Pr01bUVERUlJSAAApKSnYv39/m3aFQoGIiAjU1dXBaDTKFRoRERGRW3HowrCdxWQyISgoCAAQFBQEs9kMAKiqqkJwcLC0XHBwMKqqqqRlOyKEgKdctEOhgEtyUSoV8g/iAi0t8q8sV70mrsBc3I+n5AEwF3flKbl4Sh6A/c9glxZkHbnWpdAUiusXDlZrC2prLXKE5HL+/r1kz6VvX18MWLhX1jFcpfw/EnDxYr2sY7jiNXEV5uJ+PCUPgLm4K0/JxVPyAFo/hzvi0m9ZqtVq6VCk0WhEYGAggNYZscrKSmm5ysrK686OEREREXkKlxZkWq0Wer0eAKDX6xEbG9umXQiBkydPwtfXlwUZEREReQ3ZDlnOmzcPx44dQ01NDaKjozFr1ixkZGQgMzMT+fn5CAkJwYoVKwAAMTExOHjwIOLi4tCzZ0/k5ubKFRYRERGR25GtIFu2bNk12/Py8tq1KRQKLFq0SK5QiIiIiNwar9RPRERE1MUcKsiKi4sdaiMiIiKiG+dQQbZ06VKH2oiIiIjoxtk9h6ykpAQlJSUwm81Yt26d1H758mXYbDbZgyMiIiLyBnYLsubmZlgsFthsNjQ0NEjtvXv3xsqVK2UPjoiIiMgb2C3IIiMjERkZifHjxyM0NNRVMRERERF5FYcue9HU1IScnBwYDAZYrVapff369bIFRkREROQtHCrI5syZg0mTJmHixIlQKnmlDCIiIqLO5FBB1q1bN0yePFnuWIiIiIi8kkPTXSNHjsSmTZtgNBpRW1sr/RARERHRzXNohmznzp0AgLVr10ptCoUCRUVFTg2q1Wrh4+MDpVIJlUqFHTt2oLa2FnPnzoXBYEBoaCiWL18OPz8/p/onIiIiupU4VJAdOHCg0wfOy8tDYGCg9Fyn00Gj0SAjIwM6nQ46nQ7z58/v9HGJiIiI3I1DBZler79me0pKSqcFUlRUhA0bNkj9TpkyhQUZEREReQWHCrLTp09LjxsbG/HZZ58hLCzspgqy9PR0KBQKpKamIjU1FSaTCUFBQQCAoKAgmM3m6/ahUing79/L6RjciUql9JhcXEXu9eVJrwlzcT+ekgfAXNyVp+TiKXlcj0MFWU5OTpvn9fX1NzV7tXnzZvTr1w8mkwlpaWkYOHCgU/3YbAK1tRan43An/v69ZM+lb19fWft3NbnXlyteE1dhLu7HU/IAmIu78pRcPCUPwP7nsFMXFevRowcqKiqcDqhfv34AALVajbi4OJSWlkKtVsNoNAIAjEZjm/PLiIiIiDyZQzNk06ZNkx63tLSgrKwMY8aMcWpAi8WClpYW9O7dGxaLBYcPH8b06dOh1Wqh1+uRkZEBvV6P2NhYp/onIiIiutU4VJA9/fTT0mOVSoXQ0FAEBwc7NaDJZMKMGTMAADabDYmJiYiOjkZ4eDgyMzORn5+PkJAQrFixwqn+iYiIiG41DhVkkZGRqK6ulk7uHzBggNMD9u/fH7t27WrXHhAQgLy8PKf7JSIiIrpVOXQO2XvvvYeJEydi3759KCgokB4TERER0c1zaIZs9erVyM/Ph1qtBgCYzWZMnToVo0ePljU4IiIiIm/g0AyZEEIqxgDA398fQgjZgiIiIiLyJg7NkI0YMQLp6elISEgA0HoIMzo6WtbAiIiIiLyF3YKsoqIC1dXVWLBgAQoLC1FcXAwhBCIiIpCcnOyqGImIiIg8mt1Dlrm5ufDx8QEAxMfHIysrC9nZ2YiJiUFubq5LAiQiIiLydHYLMoPBgEGDBrVrDw8Ph8FgkC0oIiIiIm9ityBrbGzs8HdXrlzp9GCIiIiIvJHdgiw8PBxbt25t175t2zaEhYXJFhQRERGRN7F7Un92djZmzpyJ3bt3SwXYmTNn0NzcjFWrVrkkQCIiIiJPZ7cg69OnD7Zs2YIjR47g/PnzAICYmBhoNBqXBEdERETkDRy6DllUVBSioqLkjoWIiIjIKzl0pX4iIiIiko9C8B5IRERERF2KM2REREREXYwFGREREVEXY0FGRERE1MVkK8iysrKg0WiQmJgotdXW1iItLQ3x8fFIS0vDpUuXAABCCCxduhRxcXFISkrC2bNn5QqLiIiIyO3IVpBNmDABa9asadOm0+mg0WhQWFgIjUYDnU4HADh06BDKy8tRWFiIJUuWYPHixXKFRUREROR2ZCvIhg8fDj8/vzZtRUVFSElJAQCkpKRg//79bdoVCgUiIiJQV1cHo9EoV2hEREREbsWhC8N2FpPJhKCgIABAUFAQzGYzAKCqqgrBwcHScsHBwaiqqpKW7YgQAp5y0Q6FAi7JRalUyD+IC7S0yL+yXPWauAJzcT+ekgfAXNyVp+TiKXkA9j+DXVqQdeRal0JTKK5fOFitLaittcgRksv5+/eSPZe+fX0xYOFeWcdwlfL/SMDFi/WyjuGK18RVmIv78ZQ8AObirjwlF0/JA2j9HO6IS79lqVarpUORRqMRgYGBAFpnxCorK6XlKisrrzs7RkREROQpXFqQabVa6PV6AIBer0dsbGybdiEETp48CV9fXxZkRERE5DVkO2Q5b948HDt2DDU1NYiOjsasWbOQkZGBzMxM5OfnIyQkBCtWrAAAxMTE4ODBg4iLi0PPnj2Rm5srV1hEREREbke2gmzZsmXXbM/Ly2vXplAosGjRIrlCISIiInJrvFI/ERERURdzqCArLi52qI2IiIiIbpxDBdnSpUsdaiMiIiKiG2f3HLKSkhKUlJTAbDZj3bp1Uvvly5dhs9lkD46IiIjIG9gtyJqbm2GxWGCz2dDQ0CC19+7dGytXrpQ9OCIiIiJvYLcgi4yMRGRkJMaPH4/Q0FBXxURERETkVRy67EVTUxNycnJgMBhgtVql9vXr18sWGBEREZG3cKggmzNnDiZNmoSJEydCqeSVMoiIiIg6k0MFWbdu3TB58mS5YyEiIiLySg5Nd40cORKbNm2C0WhEbW2t9ENEREREN8+hGbKdO3cCANauXSu1KRQKFBUVOTWoVquFj48PlEolVCoVduzYgdraWsydOxcGgwGhoaFYvnw5/Pz8nOqfiIiI6FbiUEF24MCBTh84Ly8PgYGB0nOdTgeNRoOMjAzodDrodDrMnz+/08clIiIicjcOFWR6vf6a7SkpKZ0WSFFRETZs2CD1O2XKFBZkRERE5BUcKshOnz4tPW5sbMRnn32GsLCwmyrI0tPToVAokJqaitTUVJhMJgQFBQEAgoKCYDabr9uHSqWAv38vp2NwJyqV0mNycRW515cnvSbMxf14Sh4Ac3FXnpKLp+RxPQ4VZDk5OW2e19fX39Ts1ebNm9GvXz+YTCakpaVh4MCBTvVjswnU1lqcjsOd+Pv3kj2Xvn19Ze3f1eReX654TVyFubgfT8kDYC7uylNy8ZQ8APufw05dVKxHjx6oqKhwOqB+/foBANRqNeLi4lBaWgq1Wg2j0QgAMBqNbc4vIyIiIvJkDs2QTZs2TXrc0tKCsrIyjBkzxqkBLRYLWlpa0Lt3b1gsFhw+fBjTp0+HVquFXq9HRkYG9Ho9YmNjneqfiIiI6FbjUEH29NNPS49VKhVCQ0MRHBzs1IAmkwkzZswAANhsNiQmJiI6Ohrh4eHIzMxEfn4+QkJCsGLFCqf6JyIiIrrVOFSQRUZGorq6Wjq5f8CAAU4P2L9/f+zatatde0BAAPLy8pzul4iIiOhW5dA5ZO+99x4mTpyIffv2oaCgQHpMRERERDfPoRmy1atXIz8/H2q1GgBgNpsxdepUjB49WtbgiIiIiLyBQzNkQgipGAMAf39/CCFkC4qIiIjImzg0QzZixAikp6cjISEBQOshzOjoaFkDIyIiIvIWdguyiooKVFdXY8GCBSgsLERxcTGEEIiIiEBycrKrYiQiIiLyaHYPWebm5sLHxwcAEB8fj6ysLGRnZyMmJga5ubkuCZCIiIjI09ktyAwGAwYNGtSuPTw8HAaDQbagiIiIiLyJ3YKssbGxw99duXKl04MhIiIi8kZ2C7Lw8HBs3bq1Xfu2bdsQFhYmW1BERERE3sTuSf3Z2dmYOXMmdu/eLRVgZ86cQXNzM1atWuWSAImIiIg8nd2CrE+fPtiyZQuOHDmC8+fPAwBiYmKg0WhcEhwRERGRN3DoOmRRUVGIioqSOxYiIiIir+TQlfqJiIiISD4KwXsgEREREXUpzpARERERdTEWZERERERdjAUZERERUReTrSDLysqCRqNBYmKi1FZbW4u0tDTEx8cjLS0Nly5dAgAIIbB06VLExcUhKSkJZ8+elSssIiIiIrcjW0E2YcIErFmzpk2bTqeDRqNBYWEhNBoNdDodAODQoUMoLy9HYWEhlixZgsWLF8sVFhEREZHbka0gGz58OPz8/Nq0FRUVISUlBQCQkpKC/fv3t2lXKBSIiIhAXV0djEajXKERERERuRWHLgzbWUwmE4KCggAAQUFBMJvNAICqqioEBwdLywUHB6OqqkpatiNCCHjKRTsUCjAXN+MpeQDMxR25Mg+lUuGagWTW0iL/CvOU9xfgObl4Sh6A/W3RpQVZR651KTSF4vo7EKu1BbW1FjlCcjl//17Mxc14Sh4Ac3FHrsqjb19fDFi4V/ZxXKH8PxJw8WK9rGN4yvsL8JxcPCUPoHV77IhLv2WpVqulQ5FGoxGBgYEAWmfEKisrpeUqKyuvOztGRERE5ClcWpBptVro9XoAgF6vR2xsbJt2IQROnjwJX19fFmRERETkNWQ7ZDlv3jwcO3YMNTU1iI6OxqxZs5CRkYHMzEzk5+cjJCQEK1asAADExMTg4MGDiIuLQ8+ePZGbmytXWERERERuR7aCbNmyZddsz8vLa9emUCiwaNEiuUIhIiIicmu8Uj8RERFRF3OoICsuLnaojYiIiIhunEMF2dKlSx1qIyIiIqIbZ/ccspKSEpSUlMBsNmPdunVS++XLl2Gz2WQPjoiIiMgb2C3ImpubYbFYYLPZ0NDQILX37t0bK1eulD04IiIiIm9gtyCLjIxEZGQkxo8fj9DQUFfFRERERORVHLrsRVNTE3JycmAwGGC1WqX29evXyxYYERERkbdwqCCbM2cOJk2ahIkTJ0Kp5JUyiIiIiDqTQwVZt27dMHnyZLljISIiIvJKDk13jRw5Eps2bYLRaERtba30Q0REREQ3z6EZsp07dwIA1q5dK7UpFAoUFRU5NahWq4WPjw+USiVUKhV27NiB2tpazJ07FwaDAaGhoVi+fDn8/Pyc6p+IiIjoVuJQQXbgwIFOHzgvLw+BgYHSc51OB41Gg4yMDOh0Ouh0OsyfP7/TxyUiIiJyNw4VZHq9/prtKSkpnRZIUVERNmzYIPU7ZcoUFmRERETkFRwqyE6fPi09bmxsxGeffYawsLCbKsjS09OhUCiQmpqK1NRUmEwmBAUFAQCCgoJgNpuv24dKpYC/fy+nY3AnKpWSubgZT8kDYC7uyFPycDW515knvS6ekoun5HE9DhVkOTk5bZ7X19ff1OzV5s2b0a9fP5hMJqSlpWHgwIFO9WOzCdTWWpyOw534+/diLm7GU/IAmIs7clUeffv6yj6GK8m9zjzl/QV4Ti6ekgdgf3t06qJiPXr0QEVFhdMB9evXDwCgVqsRFxeH0tJSqNVqGI1GAIDRaGxzfhkRERGRJ3NohmzatGnS45aWFpSVlWHMmDFODWixWNDS0oLevXvDYrHg8OHDmD59OrRaLfR6PTIyMqDX6xEbG+tU/0RERES3GocKsqefflp6rFKpEBoaiuDgYKcGNJlMmDFjBgDAZrMhMTER0dHRCA8PR2ZmJvLz8xESEoIVK1Y41T8RERHRrcahgiwyMhLV1dXSyf0DBgxwesD+/ftj165d7doDAgKQl5fndL9EREREtyqHziF77733MHHiROzbtw8FBQXSYyIiIiK6eQ7NkK1evRr5+flQq9UAALPZjKlTp2L06NGyBkdERETkDRyaIRNCSMUYAPj7+0MIIVtQRERERN7EoRmyESNGID09HQkJCQBaD2FGR0fLGhgRERGRt7BbkFVUVKC6uhoLFixAYWEhiouLIYRAREQEkpOTXRUjERERkUeze8gyNzcXPj4+AID4+HhkZWUhOzsbMTExyM3NdUmARERERJ7ObkFmMBgwaNCgdu3h4eEwGAyyBUVERETkTewWZI2NjR3+7sqVK50eDBEREZE3sluQhYeHY+vWre3at23bhrCwMNmCIiIiIsBgV3kAABaCSURBVPImdk/qz87OxsyZM7F7926pADtz5gyam5uxatUqlwRIRERE5OnsFmR9+vTBli1bcOTIEZw/fx4AEBMTA41G45LgiIiIiLyBQ9chi4qKQlRUlNyxEBEREXklh67UT0RERETyUQjeA4mIiIioS3GGjIiIiKiLsSAjIiIi6mIsyIiIiIi6GAsyIiIioi7GgoyIiIioi7EgIyIiIupiLMiIiIiIuhgLMheoqanBuHHjMG7cODz88MN45JFHpOdNTU1dHZ7bx3cjBg8ejHHjxiExMRGzZ8/GTz/9ZHf5hQsXYt++fQCAKVOm4PTp064I86ZczTEhIQHJyclYt24dWlpaujqsm3bx4kXMnTsXo0aNwtixY/HMM8/g22+/vaE+6urqsGnTJpkibO+tt95CQkICkpKSMG7cOJw6deqm+3Tkfeiq9+q18tNqtTCbze2WLSoqgk6nu2Y/R48exeeff+7SODvL0aNH8Ze//KXT+ussV/cDycnJGD9+vKzr154PPvgA9957L8rKyhxavqP3zwMPPHBD42q1Wun1TkpKwv79+2/o769y9T7DHodunUQ3JyAgAO+++y4A4I033kCvXr2Qnp7exVH9H0fiE0JACAGl0jU1vNVqRbduN/727NGjh5TLc889hy1btiAtLa2zw3OKzWaDSqW66X5+nqPJZMJzzz2H+vp6zJ49u81yzq7DriCEwMyZM5GSkoL//M//BACcO3cOJpMJ//Zv/+ZwP3V1ddi8eTOefPJJuUKVlJSU4KOPPsLOnTvxq1/9CmazGc3NzbKP6yo3ml9sbCxiY2PbtVutVhw7dgy9evXC0KFDuzxOV5JzG/z5fuDjjz/GsmXLsHHjRlnGsmfPnj148MEH8d5772HWrFkuHTsvLw+BgYH45ptvkJ6ejlGjRt1wH67cZ1zPrbG39mD/9V//JW1UqampmDJlCioqKjB9+nTcd999+PLLLzFw4EC88sor6NGjh0tjq6iowIwZMzB06FCUlpbi7bffxpEjR7BmzRoIIaDVajFv3jxYrVZERUXhxIkTAIC9e/fi008/xcsvv4y9e/firbfeglKphJ+fHzZs2ACr1YrXXnsNxcXFaGxsxJ/+9CdMnDgRn376KXQ6HQICAnDhwgXs3r37puIfNmwYvvrqK/zwww+YNm0a9uzZAwBYu3YtLBaL3Z3Hnj178Pbbb0MIgZiYGMyfPx//+te/8MMPP+Bvf/sbAGDHjh04e/YscnJy8O6772LDhg1obm7G7373OyxatAgqlQoPPPAApk6dik8++QQLFizAsGHDbiqnX1Kr1ViyZAkee+wxzJo1Czt37sRHH32EpqYmWCwWrF+/HmvWrEFBQQGampoQFxeH2bNnw2KxIDMzE5WVlWhpacH06dMxduxY/OMf/8CBAwegUqkwYsQILFiwoFPj7ciRI0fQrVs3PPHEE1Lb4MGDIYTAK6+8go8//hgKhQLPPvssxo4di4aGBkyfPh11dXWwWq2YM2cORo0ahddffx3fffcdxo0bh9///veyxn/x4kUEBATgV7/6FQAgMDAQALBq1Sp8+OGHaGxsxAMPPICXXnoJCoUCU6ZMwZAhQ3D06FHU19fj5ZdfxrBhw3DlyhVkZWXhwoULuPPOO3HlyhVpjEWLFuH06dNobGzEo48+2q7ollNH+QHAxo0b8eGHH8JqtWL58uW48847sWPHDpw5cwYvvPACFi5cCD8/P3zxxRfw9/fH559/DqVSiV27diEnJ6dTt4OO4tRqtUhJSWkXp8ViwZIlS/D111/DZrNh5syZGDVqlLRtX51Vz8nJaVdAlpaW4oUXXsAbb7whbXu/7GfHjh3ttkG5Xb58GbfffjsAdLhtAMCbb76J3bt3IyQkBAEBAQgLC7upyYGGhgZ8/vnnWL9+PZ599llpn3r06FGsWrUKAQEB+PrrrxEWFoZ//OMfUCgU0t9euXIFM2bMwKOPPorHH3+8Tb/X2mc5mj8ArFu3Dtu3bwcAPPbYY5g6dWqH7a7cZ1yXIJdauXKlWLNmjRBCiFOnTomkpCRhsVhEfX29GD16tDh37pwoLy8X99xzjygpKRFCCDF//nyxbt06l8dXXl4u7r33XnHq1CkhhBA//vijGDlypDCZTKKpqUk8+eST4sCBA6K5uVk8+OCDUh979uwR2dnZQgghRo8eLS5evCiEEOLSpUtCCCE2btwo3n77bSGEEI2NjWLcuHHCYDCIw4cPi4iICGEwGJyOPyIiQgghRHNzs5g2bZrYtGmT+P7770VCQoK0zJo1a8TKlSuFEEIsWLBAFBQUCCGE+OMf/yhKS0tFZWWliImJESaTSTQ3N4spU6aIDz74QJhMJjFq1Cipn/T0dHH8+HFx4cIF8Ze//EU0NTUJIYRYtGiR2LlzpxBCiHvuuUfs3bvX6Xzs5fhzw4YNExcvXhTbt28XjzzyiKipqRFCCPHxxx+Lf//3fxctLS3CZrOJjIwMcezYMbFv3z7x/PPPS39fV1cnampqRHx8vGhpaRFC/N/r5Qp5eXni5Zdfbte+b98+MXXqVGG1WsXFixdFTEyMqKqqEs3NzaK+vl4IIaTXpaWlpd1rLafLly+L5ORkER8fLxYtWiSOHj0qhBDSuhdCiL/+9a+iqKhICNH6/vr73/8uhBDio48+Ek899ZQQQoh33nlHLFy4UAghxLlz58TgwYNFaWlpm76sVqv44x//KM6dOyf1dXUZV+c3cuRIsX79eiFE67Z8dVvfvn27ePHFF4UQrdtVRkaGsFqtQoi2+5WujvP1118Xer1eCNH6Ho+PjxcNDQ3CYrGIK1euCCGE+Pbbb8X48eOFEEIcOXJEZGRkiOLiYjF+/Hhp/9RRP7/cBuUyaNAgkZycLB599FExdOhQcfr0aSGE6HDbKC0tFcnJyeKnn34S9fX1Ii4u7qZfE71eL7KysoQQQqSmpoozZ84IIVrX2dChQ8WPP/4obDabePzxx8Xx48eFEK2vy/fffy+eeuopaT8pxP/t1zraZ/3SyJEjRWJiokhISBBDhgwRBw4cEEIIcfr0aZGYmCgaGhrE5cuXxdixY8XZs2c7bHflPuN6OEPWhU6cOIH4+Hj07NkTADBq1CgUFxdjxIgRuOOOOxAREQEASE5OxtatW6Uq35V+85vfYMiQIQCAU6dO4aGHHpL+A01MTMTx48fxyCOPdPj3Q4cOxYIFCzB69GjExcUBAA4fPoyysjLs3bsXAFBfX4+KigoAQEREBH796187He+VK1cwbtw4AK0zZI899hiMRuMN9XH69GlERkZKeSYlJeH48eMYNWoU+vfvj5MnT+K3v/0tvv32Wzz44IPYtGkTzpw5g8cee0yKQa1WAwBUKhUeffRRp/NxlPjZLWkffvhh+Pv7A2hd14cPH0ZKSgoAwGKxoLy8HMOGDcMrr7yC1157DSNHjsSwYcNgtVpx22234fnnn8cf/vAH/OEPf5A97uspLi5GQkICVCoV+vTpg+HDh+P06dOIjo7GsmXLcPz4cSiVSlRVVaG6utqlsfn4+GDHjh04ceIEjh49irlz5+K5556Dj48P1qxZgytXrqC2thZ33303tFotAEjbQFhYGAwGAwDg+PHjmDJlCgBg0KBBuPfee6UxCgoKsHXrVlitVly8eBFlZWUYNGhQl+YHAPHx8QCA+++/Hx988ME1/3706NGdcoi+s+P85JNPcODAAbzzzjsAgMbGRvz4448ICgrCSy+9hC+//BJKpRLl5eXSGGVlZXjhhRewdu1a9OvXz24/QNttUC4/P2RZUlKCBQsWYM+ePRBCXHPbKC4uRmxsrHSkZeTIkTcdw969e/HUU08BAMaOHYs9e/YgLCwMADBkyBAEBwcDaH1fGwwGaWZ0+vTp+POf/4zk5OR2fXa0zxo+fHi7Za8esvzuu+8wdepUREZGori4GKNGjUKvXr0AtG5zJ06cgBDimu1Xt013wIKsCwk793X/+dRuV7paLAIdx6tUKtv8rrGxUXq8dOlSnDp1Ch9++CHGjRuHXbt2QQiBxYsXQ6PRtOnn008/bTOeM36+k7qqW7dubU56/3l8N2rMmDEoKCjAwIEDERcXB4VCASEExo8fL30I/Nxtt90m+4fS999/D5VKJRWBv3zNMjIyMGnSpHZ/t2PHDhw8eBCvv/46Hn74YcycORP5+fn47LPPsHfvXmzcuNElh1sA4O6778b777/frr2j99zu3bthNpuxY8cOdO/eHVqt9qZeV2epVCo89NBDeOihh3DPPffgf/7nf/DVV19h+/btCAkJwRtvvNEmrquH1ZRKJWw2m9R+re39+++/xzvvvIP8/Hz4+flh4cKFLs/xl/np9XoAQPfu3QG0z+PnbnZbljPOlStXYuDAgW36eOONN9CnTx+8++67aGlpkf4RBYC+ffuisbER586dkwqyjvo5deqUS3MHWk+Ir6mpgdlsxsGDB12ybdTU1ODIkSM4f/48FAoFbDYbFAqFdErH1fc60Pr6/Hz9Dx06FIcOHUJSUlK79769fVZHfvOb30CtVqOsrKzDfYa9z1t3wW9ZdqHhw4dj//79uHLlChoaGlBUVCT9B/HDDz+gtLQUQOt/IQ8++GBXhgqgdfbq6NGjqKmpgdVqxd69exEZGSmdH1ZeXo6WlpY2/zF///33iIiIQGZmJm6//XZUVVVhxIgR+Ne//gWr1QoA+Oabb9qcN9PZ1Go1TCYTampq0NTUhI8++sju8kOGDMHx48dhNpths9mwd+9e6b+z+Ph47N+/H3v27MHYsWMBABqNBu+//z5MJhMAoLa2Vpr9kJvZbMaiRYvw5JNPXvNDfcSIEdi+fTsaGhoAAFVVVTCZTKiqqkLPnj0xbtw4pKen44svvkBDQwPq6+sRExOD7OxsfPnlly7JAQCioqLQ1NSErVu3Sm2lpaXw8/NDQUEBbDYbzGYzTpw4gSFDhqC+vh5qtRrdu3fHkSNHpPXt4+Mj5Sq3b775ps0syrlz56QvIAQEBKChoeGaReYvDR8+XDpf8uuvv8ZXX30FoPX8nJ49e8LX1xfV1dU4dOhQ5ydhx7Xyc3b2Ws7X5UbjHDFiBDZu3Ch9QH/xxRcAWmfq+/btC6VSiXfffbdNAXH77bdDp9Nh2bJlOHr0qN1+ukJZWRlsNhv8/f073DaGDh0qndvY0NBw3f3g9bz//vvSOXoHDhzAwYMHcccdd6C4uPi6fzt79mz4+/tj8eLF7X7X0T7LHpPJhB9++AG//vWvpc/Vn376CRaLBfv378ewYcM6bHflPuN6OEPWhYYMGYKEhATpUNcTTzyBe++9FxUVFbj77ruxbds2PP/88xg4cCBSU1O7OFogODgYs2fPxp/+9CcIITBy5EjpsNZf//pX/PnPf0ZISAjuuusu6XIZubm5MBgMEELg4Ycfxj333IM777wTP/74ozQlHRgYiH/+85+yxd29e3fMmDEDjz/+OO644452/9H+UlBQEObNm4ennnoKQghER0dLJ8X6+fnhrrvuwoULF6T/oO+66y5kZmbi6aefRktLC7p3744XXngBoaGhsuRz9bCs1WqFSqXCuHHjOvwm6YgRI1BWVib9t9mrVy+89tprqKiowKuvvgqlUolu3bph8eLF0snAV/+bzsrKkiX+a1EoFFi1ahVyc3Oh0+lw2223ITQ0FNnZ2WhoaMC4ceOgUCgwf/589O3bF0lJSXj22WcxYcIEDB48WHpNAwICMHToUCQmJuKRRx6R9QRdi8WCpUuXoq6uDiqVCr/97W/x0ksvwdfXF0lJSQgNDUV4ePh1+3niiSeQlZWFpKQkDB48WHpfDRo0CPfddx8SEhLQv39/Wb6haE9H+TnzQT5y5EjMnj0bRUVFnX5S/43GOX36dOTm5iI5ORlCCISGhuLtt9/G5MmTMWvWLOzbtw8PPfSQdGjrqj59+mD16tV45plnkJub22E/rvLz0zPE///yi0ql6nDbGDJkCLRaLZKTkxEaGor7778fvr6+To+/d+9ePPPMM23a4uPjsXv3bumfVXuef/55ZGdn49VXX5Vm1YCO91lXjwD83FNPPQWlUgmr1YrnnnsOffr0QZ8+fTBhwgRMnDgRQOvJ+/fddx8AdNjuqn3G9SjErTCP52UqKiowe/bsdofeiIiInNXQ0AAfHx/89NNPePLJJ7FkyRLpnC/qepwhIyIi8gIvvPACLly4gMbGRowfP57FmJvhDBkRERFRF+NJ/URERERdjAUZERERURdjQUZERETUxViQEdEt5eLFi5g7dy5GjRqFsWPH4plnnsG33357zWXr6uqwadMml8S1efNm6YKkREQ3iif1E9EtQwiBSZMmISUlRboR+blz59DQ0HDNa1v98sbycrFarejWjV9aJyLncQ9CRLeMI0eOoFu3blIxBgCDBw9GQ0MDnnrqKdTV1cFqtWLOnDkYNWoUXn/9dXz33XcYN24cfv/732PBggVYs2YNCgoK0NTUhLi4OMyePRsA8Oabb2L37t0ICQlBQEAAwsLCkJ6ejnPnzmHRokX46aef8Jvf/Aa5ubnw8/PDlClT8MADD+Dzzz+HVqtFQ0MDevXqhfT0dHz33Xd48cUXUVNTgx49emDJkiW48847UVBQgDfffBNKpRK+vr4um70jIvfHgoyIbhnnz5+/5rWTbrvtNrz55pvo3bs3zGYzUlNTERsbi+eeew7nz5+XLrL8ySefoKKiAvn5+RBC4Nlnn8Xx48fRo0cPFBYWQq/Xw2q1YsKECdI4f/vb35CTk4PIyEisWLECq1atwvPPPw+g9ZDoxo0bAbTeC/GqnJwcvPjiixgwYABOnTqFF198EevXr8c///lP6QbVdXV1cq8uIrqFsCAjolueEALLli3D8ePHoVQqUVVVherq6nbLHT58GIcPH5Zu22WxWFBeXo6GhgbExsaiR48eAFpv9QO03t+wvr4ekZGRAIDx48djzpw5Un/XukVMQ0MDSkpK2ix39VZiDzzwABYuXIgxY8YgLi6uk7InIk/AgoyIbhl33333NW/YvXv3bpjNZuzYsQPdu3eHVquV7sn5c0IIZGRkSPfJu+q///u/nYqnZ8+e1xzj9ttvv+atz1566SWcOnUKH330EVJSUqDX6xEQEODU2ETkWfgtSyK6ZURFRaGpqQlbt26V2kpLS/G///u/UKvV6N69O44cOQKDwQAA8PHxQUNDg7TsiBEjsH37dqmtqqoKJpMJQ4cOxYcffojGxkY0NDRIN6b29fXF7bffjhMnTgAA3n33XQwfPtxujL1798Ydd9yBgoICAK0F2pdffgkA+O677/C73/0Oc+bMQUBAACorKztnxRDRLY8zZER0y1AoFFi1ahVyc3Oh0+lw2223ITQ0FDNnzsTLL7+MCRMmYPDgwRg4cCAAICAgAEOHDkViYiIeeeQRLFiwAGVlZdIMWa9evfDaa69hyJAh0Gq1SE5ORmhoKO6//374+voCAF555RXppP7+/fvj73//+3XjfO2117B48WK89dZbsFqtGDt2LAYNGoRXX30VFRUVEEIgKioKgwYNkm9lEdEthZe9ICJC67lfPj4++Omnn/Dkk09iyZIlvPkyEbkMZ8iIiAC88MILuHDhAhobGzF+/HgWY0TkUpwhIyIiIupiPKmfiIiIqIuxICMiIiLqYizIiIiIiLoYCzIiIiKiLsaCjIiIiKiL/T/w2mq5K2Uh2wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x288 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net.eval()\n",
    "fmnist_outputs = net(fmnist_sample, True).max(1, keepdim=True)[1].detach().cpu().numpy()\n",
    "for _ in range(99):\n",
    "    fmnist_outputs = np.append(fmnist_outputs, net(fmnist_sample, True).max(1, keepdim=True)[1].detach().cpu().numpy(), axis=1)\n",
    "sns.set_style(\"darkgrid\")\n",
    "plt.subplots(5,1,figsize=(10,4))\n",
    "for i in range(5):\n",
    "    plt.subplot(5,1,i+1)\n",
    "    plt.ylim(0,100)\n",
    "    plt.xlabel(\"Categories\")\n",
    "    plt.xticks(range(10), [\"Top\", \"Trouser\", \"Pullover\", \"Dress\", \"Coat\", \"Sandal\", \"Shirt\", \"Sneaker\", \"Bag\", \"Ankle Boot\"])\n",
    "    plt.ylabel(\"Count\")\n",
    "    plt.yticks(range(50,101,50))\n",
    "    plt.hist(fmnist_outputs[i], np.arange(-0.5, 10, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Out-of-domain uncertainty "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise \n",
    "\n",
    "Use some MNIST samples as OOD data to test the behavior of our BNN similarly as above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
